Malicious-URL-Detection-ML / OldCode / 3-DictonaryWord.py
3-DictonaryWord.py
Raw
import re
import requests

def tokenize_url(url):
    tokens = re.findall(r'[a-zA-Z0-9]+', url)
    new_tokens = []
    for token in tokens:
        new_tokens.append(token)
        if token.find(".") >= 0:
            dot_split = token.split('.')
            if "com" in dot_split:
                dot_split.remove("com")
            if "www" in dot_split:
                dot_split.remove("www")
            new_tokens.extend(dot_split)
    return new_tokens

def check_dictionary(words):
    english_words = requests.get("https://raw.githubusercontent.com/dwyl/english-words/master/words.txt").text.splitlines()
    return [int(word.lower() in english_words) for word in words]

url = input("Enter a URL: ")
tokens = tokenize_url(url)
words_in_dictionary = check_dictionary(tokens)

print("Tokenized URL:", tokens)
print("Words in English Dictionary:", words_in_dictionary)