im using beautifulSoup4, this is my code:
def extract(page):
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
r = requests.get(url)
soup = BeautifulSoup(r.content,'html.parser')
return soup
def transform(soup):
divs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
for items in divs:
location = items.find('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()
salary = items.find_next_sibling('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()
both span have the same class, but when i scrapped, both results were same.
CodePudding user response：
get the all spans by find_all and then remove the items that they have less than 2 spans. then get location and salary from the spans list
import requests
from bs4 import BeautifulSoup
def extract(page):
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
r = requests.get(url)
soup = BeautifulSoup(r.content,'html.parser')
# transform
jobs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
data = []
for job in jobs[:100]:
items = job.find_all('span', attrs = {'class' : 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'})
if len(items) < 2:
continue
location = items[0].text.strip()
salary = items[1].text.strip()
data.append([location, salary])
return data
data = extract(1)
print(data)
CodePudding user response：
Try to avoid selecting by dynamic parts in HTML such as classes - While you still figured out the sibling relation you could go with find_next_sibling('span').
Following will select only by tags and also checks if there is a salary or not to avoid errors but scrape the whole job, while display salary None:
for item in soup.select('article'):
data.append({
'title': item.h1.text,
'location': item.h1.parent.next_sibling.text,
'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
})
Example
import requests
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/1/'
r = requests.get(url)
soup = BeautifulSoup(r.content, 'html.parser')
data=[]
for item in soup.select('article'):
data.append({
'title': item.h1.text,
'location': item.h1.parent.next_sibling.text,
'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
})
data
CodePudding user response：
Using dynamic classes is not the best idea. The site uses api for job search as well as graphql.
import requests
import json
import pandas as pd
def foo(page, keyword):
url = "https://xapi.supercharge-srp.co/job-search/graphql?country=my&isSmartSearch=true"
payload = json.dumps({
"query": "query getJobs($country: String, $locale: String, $keyword: String, $createdAt: String, $jobFunctions: [Int], $categories: [String], $locations: [Int], $careerLevels: [Int], $minSalary: Int, $maxSalary: Int, $salaryType: Int, $candidateSalary: Int, $candidateSalaryCurrency: String, $datePosted: Int, $jobTypes: [Int], $workTypes: [String], $industries: [Int], $page: Int, $pageSize: Int, $companyId: String, $advertiserId: String, $userAgent: String, $accNums: Int, $subAccount: Int, $minEdu: Int, $maxEdu: Int, $edus: [Int], $minExp: Int, $maxExp: Int, $seo: String, $searchFields: String, $candidateId: ID, $isDesktop: Boolean, $isCompanySearch: Boolean, $sort: String, $sVi: String, $duplicates: String, $flight: String, $solVisitorId: String) {\n jobs(\n country: $country\n locale: $locale\n keyword: $keyword\n createdAt: $createdAt\n jobFunctions: $jobFunctions\n categories: $categories\n locations: $locations\n careerLevels: $careerLevels\n minSalary: $minSalary\n maxSalary: $maxSalary\n salaryType: $salaryType\n candidateSalary: $candidateSalary\n candidateSalaryCurrency: $candidateSalaryCurrency\n datePosted: $datePosted\n jobTypes: $jobTypes\n workTypes: $workTypes\n industries: $industries\n page: $page\n pageSize: $pageSize\n companyId: $companyId\n advertiserId: $advertiserId\n userAgent: $userAgent\n accNums: $accNums\n subAccount: $subAccount\n minEdu: $minEdu\n edus: $edus\n maxEdu: $maxEdu\n minExp: $minExp\n maxExp: $maxExp\n seo: $seo\n searchFields: $searchFields\n candidateId: $candidateId\n isDesktop: $isDesktop\n isCompanySearch: $isCompanySearch\n sort: $sort\n sVi: $sVi\n duplicates: $duplicates\n flight: $flight\n solVisitorId: $solVisitorId\n ) {\n total\n totalJobs\n relatedSearchKeywords {\n keywords\n type\n totalJobs\n }\n solMetadata\n suggestedEmployer {\n name\n totalJobs\n }\n queryParameters {\n key\n searchFields\n pageSize\n }\n experiments {\n flight\n }\n jobs {\n id\n adType\n sourceCountryCode\n isStandout\n companyMeta {\n id\n advertiserId\n isPrivate\n name\n logoUrl\n slug\n }\n jobTitle\n jobUrl\n jobTitleSlug\n description\n employmentTypes {\n code\n name\n }\n sellingPoints\n locations {\n code\n name\n slug\n children {\n code\n name\n slug\n }\n }\n categories {\n code\n name\n children {\n code\n name\n }\n }\n postingDuration\n postedAt\n salaryRange {\n currency\n max\n min\n period\n term\n }\n salaryVisible\n bannerUrl\n isClassified\n solMetadata\n }\n }\n}\n",
"variables": {
"keyword": keyword,
"jobFunctions": [],
"locations": [],
"salaryType": 1,
"jobTypes": [],
"createdAt": None,
"careerLevels": [],
"page": page,
"country": "my",
"categories": [],
"workTypes": [],
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"industries": [],
"locale": "en"
}
})
headers = {
'accept': '*/*',
'content-type': 'application/json'
}
response = requests.request("POST", url, headers=headers, data=payload)
salary = "Not specified"
result = []
for job in (response.json()['data']['jobs']['jobs']):
locations = []
if job['salaryVisible']:
salary = f"{job['salaryRange']['min']}-{job['salaryRange']['max']} {job['salaryRange']['currency']}"
for location in job['locations']:
locations.append(location['name'])
result.append([job['jobTitle'], job['description'], job['jobUrl'], salary, ", ".join(locations)])
df = pd.DataFrame(result)
df.columns = ['Title', 'Description', 'Link', 'Salary', 'Location']
print(df.to_string())
foo(1, "personal assistant")
OUTPUT:
Title Description Link Salary Location
0 Personal Assistant/ Secretary Job Description:GETO GLOBAL CONSTRUCTION TECH MALAYSIA SDN. BHD. is a subsidiary of a public listed company from China, specializing in Aluminum... https://www.jobstreet.com.my/en/job/personal-assistant-secretary-4986870?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=1&jobId=jobstreet-my-job-4986870 3000-6000 MYR Shah Alam/Subang
1 Executive Assistant Responsibilities: Responsible in managing day-to-day office related and administrative tasks. Organizing and attending meetings and ensuring well... https://www.jobstreet.com.my/en/job/executive-assistant-4986449?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=2&jobId=jobstreet-my-job-4986449 2000-3000 MYR Johor - Others
2 Secretary A good write-up Secretary's resume is a Must with experience and can handle Managing Director company matters only. Fresh grads or many years in... https://www.jobstreet.com.my/en/job/secretary-4986714?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=3&jobId=jobstreet-my-job-4986714 2000-3000 MYR Petaling Jaya
3 Secretary/Personal assistant We are looking for Secretary/Personal assistant in our company for: Organising and maintaining office systems Screening phone calls, emails, enquiries... https://www.jobstreet.com.my/en/job/secretary-personal-assistant-4983131?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=4&jobId=jobstreet-my-job-4983131 2000-3000 MYR Kuala Lumpur
4 Personal Assistant to Country Leader Do you want to make a positive impact on the world? Sustainable development is at the heart of everything we do.Seventy-five years ago, we created a... https://www.jobstreet.com.my/en/job/personal-assistant-to-country-leader-4986906?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=5&jobId=jobstreet-my-job-4986906 2000-3000 MYR Petaling Jaya
5 Personal Assistant to Senior Director Requirement: 5 – 8 years in handling PA end to end tasks for a Director/CEO/Managing Director Highly proficient in MS Office software Strong knowledge... https://www.jobstreet.com.my/en/job/personal-assistant-to-senior-director-4986792?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=6&jobId=jobstreet-my-job-4986792 2000-3000 MYR Shah Alam/Subang
6 Personal Assistant Report to top management; Attend to all aspects of PA and corporate work; Draft and manage correspondences, email, and memos; Schedule and organize... https://www.jobstreet.com.my/en/job/personal-assistant-4987086?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=7&jobId=jobstreet-my-job-4987086 2000-4000 MYR Puchong
7 Personal Assistant (Secretary) to Director Responsibilities: Provide support to Director and act as the primary liaison with various divisions/department or any newly set-up branch within... https://www.jobstreet.com.my/en/job/personal-assistant-secretary-to-director-4980388?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=8&jobId=jobstreet-my-job-4980388 2000-4000 MYR Kuala Lumpur, Selangor
8 Personal Assistant Personal Assistants perform a variety of administrative tasks for an individual.Here are some common responsibilities of a Personal Assistant:... https://www.jobstreet.com.my/en/job/personal-assistant-4986053?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=9&jobId=jobstreet-my-job-4986053 2000-4000 MYR Subang Jaya
9 Special Officer to CEO - Public Listed Conglomerate Reporting and supporting the CEO in his daily work in wide variety of area - including preparation for meetings, development of presentation materials... https://www.jobstreet.com.my/en/job/special-officer-to-ceo-public-listed-conglomerate-4986807?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=10&jobId=jobstreet-my-job-4986807 15000-20000 MYR Kuala Lumpur
...