In my project I'm trying to scrape reviews of a car chose by user. In the django app I want to pass the data in the context dictionary to an another scrape.py
file.
Code:
models.py
class SelectCar(models.Model):
BRANDS = (
('BMW','BMW'),
('MR','Mercedes')
)
brand = models.CharField(max_length=30,
choices=BRANDS,
default="BMW")
CAR_MODELS = (
('X1','X1'),
('X2','X2'),
('X3','X3')
)
car_model = models.CharField(max_length=30,
choices=CAR_MODELS,
default="X1")
forms.py
from .models import SelectCar
class SelectCarForm(forms.ModelForm):
class Meta:
model = SelectCar
fields = '__all__'
scrape.py, here i want to import the context data and choose the url for the specific car in an if-else chain. For example, if chose car is honda amaze, context should be imported into scrape.py
such that I'm able to choose url.
import requests
from bs4 import BeautifulSoup
"""import context here in some way"""
def happyScrape():
"""here, import the context and choose url based on chosen car
if car is Honda amaze, choose url for honda amaze and so on.
example- if brand is honda and model is amaze,
"""
url = 'https://www.carwale.com/honda-cars/amaze/user-reviews/'\
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'}
cw_result = requests.get(url, headers=headers)
cw_soup = BeautifulSoup(cw_result.text, 'html5lib')
scrapeList =[]
#scrape code here, stored in scrapeListHTML
for data in scrapeListHtml:
scrapeList.append(data.text)
with open(r'./scrape/scrapeListFile.txt', 'w') as f:
for item in scrapeList:
# write each item on a new line
f.write("%s\n" % item)
views.py
from .forms import SelectCarForm
def ScrapeView(request):
context = {}
context['SelectCarForm'] = SelectCarForm()
if request.method == 'POST' and 'run_script' in request.POST:
from .scrape import happyScrape
happyScrape()
return render(request, "scrape.html", context)
EDIT solved it by passing data to happyScrape() itself in the form of brand, car_model as the answer said.
CodePudding user response:
Pass the specific data from the context
in the happyScrape()
function.
In scrape.py
def happyScrape(data):
# use this data here
...
In views.py
def ScrapeView(request):
...
happyScrape(<data_from_context>)
...