import re
import requests
def tokenize_url(url):
tokens = re.findall(r'[a-zA-Z0-9]+', url)
new_tokens = []
for token in tokens:
new_tokens.append(token)
if token.find(".") >= 0:
dot_split = token.split('.')
if "com" in dot_split:
dot_split.remove("com")
if "www" in dot_split:
dot_split.remove("www")
new_tokens.extend(dot_split)
return new_tokens
def check_dictionary(words):
english_words = requests.get("https://raw.githubusercontent.com/dwyl/english-words/master/words.txt").text.splitlines()
return [int(word.lower() in english_words) for word in words]
url = input("Enter a URL: ")
tokens = tokenize_url(url)
words_in_dictionary = check_dictionary(tokens)
print("Tokenized URL:", tokens)
print("Words in English Dictionary:", words_in_dictionary)
Enter a URL: https://colab.research.google.com/drive/18z-3vrAoPD_AOWBNzDhypMeXli-uzHoA#scrollTo=5BchG6xaKEIM
Tokenized URL: ['https', 'colab', 'research', 'google', 'com', 'drive', '18z', '3vrAoPD', 'AOWBNzDhypMeXli', 'uzHoA', 'scrollTo', '5BchG6xaKEIM']
Words in English Dictionary: [0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0]