I have a working web scraper written in Python, Selenium and Chromedriver (all up-to-date version wise) and various other software packages. The target webpage has a field for the phone number, but you have to click a button to reveal it:
Notice that the class names are all gobbledegook and have a tendency to change on me so I am trying to use names that will stay constant. The only thing I can think of is to use the TAG "button" and the ID 'p' with the attribute 'aria-label="Reveal phone number"'. I cannot figure out a CSS_SELECTOR that I can use with these values to do a fluent wait after I make sure that field is on the webpage. Here is the code I have tried. I know this is not enough to test, but I am hoping for some ideas on what else to try:
results = soup.find_all('p',attrs={'aria-label':'Reveal phone number'})
if re.search('Reveal phone number',str(results)):
elems = ''
try:
elems = webdwait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,"[aria-label='Reveal phone number']")))
except Exception as err:
logging.info(f"ERROR F{flag}-70: Webpage element (PhoneRevealButton) search failed ...")
logging.info(f"ERROR F{flag}-70: type={type(err)}: {err}")
print(f" WARNING F{flag}-70: Webpage search failed ({type(err)}): trying again ...",flush=True)
failed = True
if not failed:
for elem in elems:
if re.search('Reveal phone number',str(elem.get_attribute('outerHTML'))):
driver.execute_script("arguments[0].scrollIntoView(true);",elem)
try:
driver.execute_script("arguments[0].click();",elem)
except Exception as err:
logging.info(f"ERROR F{flag}-71: Webpage element (PhoneRevealButton) search failed ...")
logging.info(f"ERROR F{flag}-71: type={type(err)}: {err}")
print(f" WARNING F{flag}-71: Webpage click failed ({type(err)}): trying again ...",flush=True)
failed = True
if not failed:
# Wait until these elements are updated
lcnt = 0
while True:
try:
elems = webdwait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,"[aria-label='Reveal phone number']")))
except Exception as err:
# Element is gone?
logging.info(f"INFO F{flag}-72: type={type(err)}: {err}")
break
cnt = 0
for elem in elems:
if re.search('Reveal phone number',str(elem.get_attribute('outerHTML'))):
# Element still exists
cnt += 1
if cnt == 0:
break
lcnt += 1
if lcnt * timeout_wait2 >= timeout_dri:
print(f" ERROR: Waited too long for webpage to update - phone number not retrieved")
break
time.sleep(timeout_wait2)