{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"collapsed_sections": [
"xJJatsUev-We"
]
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"source": [
"import pandas as pd\n",
"import re"
],
"metadata": {
"id": "40uCf910RUdV"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df = pd.read_excel('/content/data_en_it_tagged.xlsx')"
],
"metadata": {
"id": "w3Iur8DPVlHe"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 424
},
"id": "9LaOvu7jVpYL",
"outputId": "02e3a47b-b776-4226-e003-56c94fa73709"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" direction id text_type \\\n",
"0 en_to_it 0001en_sp_st en_sp_st \n",
"1 en_to_it 0002en_sp_st en_sp_st \n",
"2 en_to_it 0003en_sp_st en_sp_st \n",
"3 en_to_it 0004en_sp_st en_sp_st \n",
"4 en_to_it 0005en_sp_st en_sp_st \n",
".. ... ... ... \n",
"523 it_to_en 1064en_wr_tt en_wr_tt \n",
"524 it_to_en 1065en_wr_tt en_wr_tt \n",
"525 it_to_en 1066en_wr_tt en_wr_tt \n",
"526 it_to_en 1067en_wr_tt en_wr_tt \n",
"527 it_to_en 1068en_wr_tt en_wr_tt \n",
"\n",
" text \n",
"0 Thank/VV you/PP President/NP ./SENT Well/RB... \n",
"1 Thank/VV you/PP very/RB much/JJ Mr/NP Pre... \n",
"2 Excuse/VV me/PP ./SENT Thank/VV you/PP Pre... \n",
"3 President/NP ,/, the/DT upheaval/NN in/IN ... \n",
"4 Thank/VV you/PP Mr/NP President/NP ./SENT ... \n",
".. ... \n",
"523 Mr/NP President/NP ,/, ladies/NNS and/CC g... \n",
"524 Mr/NP President/NP ,/, High/NP Representati... \n",
"525 Mr/NP President/NP ,/, ladies/NNS and/CC g... \n",
"526 Mr/NP President/NP ,/, ladies/NNS and/CC g... \n",
"527 Mr/NP President/NP ,/, ladies/NNS and/CC g... \n",
"\n",
"[528 rows x 4 columns]"
],
"text/html": [
"\n",
" <div id=\"df-0650964d-e86d-423e-a3ed-9f013d66d5ee\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>direction</th>\n",
" <th>id</th>\n",
" <th>text_type</th>\n",
" <th>text</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>en_to_it</td>\n",
" <td>0001en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>Thank/VV you/PP President/NP ./SENT Well/RB...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>en_to_it</td>\n",
" <td>0002en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>Thank/VV you/PP very/RB much/JJ Mr/NP Pre...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>en_to_it</td>\n",
" <td>0003en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>Excuse/VV me/PP ./SENT Thank/VV you/PP Pre...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>en_to_it</td>\n",
" <td>0004en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>President/NP ,/, the/DT upheaval/NN in/IN ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>en_to_it</td>\n",
" <td>0005en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>Thank/VV you/PP Mr/NP President/NP ./SENT ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>523</th>\n",
" <td>it_to_en</td>\n",
" <td>1064en_wr_tt</td>\n",
" <td>en_wr_tt</td>\n",
" <td>Mr/NP President/NP ,/, ladies/NNS and/CC g...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>524</th>\n",
" <td>it_to_en</td>\n",
" <td>1065en_wr_tt</td>\n",
" <td>en_wr_tt</td>\n",
" <td>Mr/NP President/NP ,/, High/NP Representati...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>525</th>\n",
" <td>it_to_en</td>\n",
" <td>1066en_wr_tt</td>\n",
" <td>en_wr_tt</td>\n",
" <td>Mr/NP President/NP ,/, ladies/NNS and/CC g...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>526</th>\n",
" <td>it_to_en</td>\n",
" <td>1067en_wr_tt</td>\n",
" <td>en_wr_tt</td>\n",
" <td>Mr/NP President/NP ,/, ladies/NNS and/CC g...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>527</th>\n",
" <td>it_to_en</td>\n",
" <td>1068en_wr_tt</td>\n",
" <td>en_wr_tt</td>\n",
" <td>Mr/NP President/NP ,/, ladies/NNS and/CC g...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>528 rows × 4 columns</p>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0650964d-e86d-423e-a3ed-9f013d66d5ee')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-0650964d-e86d-423e-a3ed-9f013d66d5ee button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-0650964d-e86d-423e-a3ed-9f013d66d5ee');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {},
"execution_count": 5
}
]
},
{
"cell_type": "code",
"source": [
"# Function to remove tags from text\n",
"def remove_tags(text):\n",
" ignore_tags = ['SENT', ',', '\\'', 'DYSF', ':', 'EPAUSE', '``', 'FPAUSE', 'UNCLEAR', 'NOCAT', 'SYM', 'PUN', '/NOCAT']\n",
" words = text.split()\n",
" cleaned_words = []\n",
" for word in words:\n",
" try:\n",
" if word.split('/')[1] not in ignore_tags:\n",
" cleaned_words.append(word.split('/')[0])\n",
" except IndexError: # word does not have a '/'\n",
" cleaned_words.append(word)\n",
" return ' '.join(cleaned_words)\n",
"\n",
"# Apply function to remove tags from \"text\"\n",
"df['text'] = df['text'].apply(remove_tags)\n",
"\n",
"# Function to split texts into chunks of 1000 tokens\n",
"def split_text(text):\n",
" words = text.split()\n",
" return [words[i:i+1000] for i in range(0, len(words), 1000)]\n",
"\n",
"# Function to handle rows merging/splitting\n",
"def handle_rows(df):\n",
" rows = []\n",
" buffer_text = []\n",
" buffer_id = []\n",
"\n",
" for index, row in df.iterrows():\n",
" chunks = split_text(row['text'])\n",
" if len(buffer_text) > 0 and buffer_id[-1].split('_')[-1] != row['id'].split('_')[-1]: # Check if text_type has changed\n",
" if len(buffer_text) >= 1000:\n",
" rows.append({'direction': row['direction'], 'id': ', '.join(buffer_id), 'text_type': row['text_type'], 'text': ' '.join(buffer_text[:1000])})\n",
" buffer_text = []\n",
" buffer_id = []\n",
" for i, chunk in enumerate(chunks):\n",
" if i == 0 and len(buffer_text) < 1000:\n",
" buffer_text += chunk\n",
" buffer_id.append(row['id'])\n",
" if len(buffer_text) >= 1000:\n",
" rows.append({'direction': row['direction'], 'id': ', '.join(buffer_id), 'text_type': row['text_type'], 'text': ' '.join(buffer_text[:1000])})\n",
" buffer_text = buffer_text[1000:]\n",
" buffer_id = [row['id']]\n",
" else:\n",
" buffer_text = chunk\n",
" buffer_id = [row['id']]\n",
" if len(buffer_text) >= 1000:\n",
" rows.append({'direction': row['direction'], 'id': ', '.join(buffer_id), 'text_type': row['text_type'], 'text': ' '.join(buffer_text[:1000])})\n",
" buffer_text = buffer_text[1000:]\n",
" return pd.DataFrame(rows)\n",
"\n",
"# Apply function to handle rows merging/splitting\n",
"df = handle_rows(df)\n",
"\n",
"# Print the number of tokens for each unique value in the text_type column\n",
"token_counts = df.groupby('text_type')['text'].apply(lambda x: sum(len(text.split()) for text in x))\n",
"print(token_counts)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "NvfBC4F_vXdL",
"outputId": "1b27d368-720e-4acd-f609-8c959d6542a7"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"text_type\n",
"en_sp_st 20000\n",
"en_sp_tt 16000\n",
"en_wr_st 19000\n",
"en_wr_tt 18000\n",
"it_sp_st 17000\n",
"it_sp_tt 17000\n",
"it_wr_st 17000\n",
"it_wr_tt 18000\n",
"Name: text, dtype: int64\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"def calculate_sttr(text):\n",
"\n",
" # lowercase and tokenize the text\n",
" tokens = text.lower().split()\n",
"\n",
" # number of tokens (total number of words)\n",
" num_tokens = len(tokens)\n",
"\n",
" # number of types (unique words)\n",
" num_types = len(set(tokens))\n",
"\n",
" # calculate Type-Token Ratio (TTR)\n",
" ttr = num_types / num_tokens if num_tokens > 0 else 0\n",
"\n",
" return ttr\n",
"\n",
"# apply the function to the 'text' column and create a new 'ttr' column\n",
"df['sttr'] = df['text'].apply(calculate_sttr)"
],
"metadata": {
"id": "LtIT3tv9GGid"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df = df[['direction', 'id', 'text_type', 'sttr', 'text']]"
],
"metadata": {
"id": "CmjUKuzBGUDO"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 424
},
"id": "ZBPNYPe7GITS",
"outputId": "8622cc0f-eb8b-4ccc-892a-e452e8196d06"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" direction id text_type \\\n",
"0 en_to_it 0001en_sp_st, 0002en_sp_st, 0003en_sp_st en_sp_st \n",
"1 en_to_it 0003en_sp_st, 0004en_sp_st, 0005en_sp_st, 0006... en_sp_st \n",
"2 en_to_it 0006en_sp_st, 0007en_sp_st, 0008en_sp_st en_sp_st \n",
"3 en_to_it 0008en_sp_st, 0009en_sp_st, 0010en_sp_st en_sp_st \n",
"4 en_to_it 0010en_sp_st, 0011en_sp_st, 0012en_sp_st, 0013... en_sp_st \n",
".. ... ... ... \n",
"137 it_to_en 1045en_wr_tt, 1046en_wr_tt, 1047en_wr_tt, 1048... en_wr_tt \n",
"138 it_to_en 1049en_wr_tt, 1050en_wr_tt, 1051en_wr_tt, 1052... en_wr_tt \n",
"139 it_to_en 1055en_wr_tt, 1056en_wr_tt, 1057en_wr_tt, 1058... en_wr_tt \n",
"140 it_to_en 1059en_wr_tt, 1060en_wr_tt, 1061en_wr_tt, 1062... en_wr_tt \n",
"141 it_to_en 1063en_wr_tt, 1064en_wr_tt, 1065en_wr_tt, 1066... en_wr_tt \n",
"\n",
" sttr text \n",
"0 0.461924 Thank you President Well some colleagues took ... \n",
"1 0.478478 refrain from using violence and that there wil... \n",
"2 0.431156 their hard work their thoughtfulness and commi... \n",
"3 0.424000 into our Committee to do that I think has been... \n",
"4 0.427711 that these measures when endorsed and adopted ... \n",
".. ... ... \n",
"137 0.439759 regions and between administrative structures ... \n",
"138 0.442000 by the European Union are those specifically i... \n",
"139 0.463928 couples around the world who every day face th... \n",
"140 0.483903 We are now about to adopt the agreement on Ira... \n",
"141 0.458753 book Premesse della politica Premises of pol... \n",
"\n",
"[142 rows x 5 columns]"
],
"text/html": [
"\n",
" <div id=\"df-4fe5e669-2f81-4ac6-8b20-635af14803e5\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>direction</th>\n",
" <th>id</th>\n",
" <th>text_type</th>\n",
" <th>sttr</th>\n",
" <th>text</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>en_to_it</td>\n",
" <td>0001en_sp_st, 0002en_sp_st, 0003en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>0.461924</td>\n",
" <td>Thank you President Well some colleagues took ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>en_to_it</td>\n",
" <td>0003en_sp_st, 0004en_sp_st, 0005en_sp_st, 0006...</td>\n",
" <td>en_sp_st</td>\n",
" <td>0.478478</td>\n",
" <td>refrain from using violence and that there wil...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>en_to_it</td>\n",
" <td>0006en_sp_st, 0007en_sp_st, 0008en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>0.431156</td>\n",
" <td>their hard work their thoughtfulness and commi...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>en_to_it</td>\n",
" <td>0008en_sp_st, 0009en_sp_st, 0010en_sp_st</td>\n",
" <td>en_sp_st</td>\n",
" <td>0.424000</td>\n",
" <td>into our Committee to do that I think has been...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>en_to_it</td>\n",
" <td>0010en_sp_st, 0011en_sp_st, 0012en_sp_st, 0013...</td>\n",
" <td>en_sp_st</td>\n",
" <td>0.427711</td>\n",
" <td>that these measures when endorsed and adopted ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>137</th>\n",
" <td>it_to_en</td>\n",
" <td>1045en_wr_tt, 1046en_wr_tt, 1047en_wr_tt, 1048...</td>\n",
" <td>en_wr_tt</td>\n",
" <td>0.439759</td>\n",
" <td>regions and between administrative structures ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>138</th>\n",
" <td>it_to_en</td>\n",
" <td>1049en_wr_tt, 1050en_wr_tt, 1051en_wr_tt, 1052...</td>\n",
" <td>en_wr_tt</td>\n",
" <td>0.442000</td>\n",
" <td>by the European Union are those specifically i...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>139</th>\n",
" <td>it_to_en</td>\n",
" <td>1055en_wr_tt, 1056en_wr_tt, 1057en_wr_tt, 1058...</td>\n",
" <td>en_wr_tt</td>\n",
" <td>0.463928</td>\n",
" <td>couples around the world who every day face th...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>140</th>\n",
" <td>it_to_en</td>\n",
" <td>1059en_wr_tt, 1060en_wr_tt, 1061en_wr_tt, 1062...</td>\n",
" <td>en_wr_tt</td>\n",
" <td>0.483903</td>\n",
" <td>We are now about to adopt the agreement on Ira...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>141</th>\n",
" <td>it_to_en</td>\n",
" <td>1063en_wr_tt, 1064en_wr_tt, 1065en_wr_tt, 1066...</td>\n",
" <td>en_wr_tt</td>\n",
" <td>0.458753</td>\n",
" <td>book Premesse della politica Premises of pol...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>142 rows × 5 columns</p>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4fe5e669-2f81-4ac6-8b20-635af14803e5')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-4fe5e669-2f81-4ac6-8b20-635af14803e5 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-4fe5e669-2f81-4ac6-8b20-635af14803e5');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {},
"execution_count": 110
}
]
},
{
"cell_type": "code",
"source": [
"# calculate the mean and median of ttr for each unique text_type\n",
"mean_ttr = df.groupby('text_type')['sttr'].mean()\n",
"median_ttr = df.groupby('text_type')['sttr'].median()\n",
"\n",
"# print the mean and median of ttr for each unique text_type\n",
"print(\"Mean TTR by Text Type:\\n\", mean_ttr, \"\\n\")\n",
"print(\"Median TTR by Text Type:\\n\", median_ttr)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "T_xS5zqVG0Wc",
"outputId": "24ad2935-a2a7-447d-aa93-045b7f8a3a66"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Mean TTR by Text Type:\n",
" text_type\n",
"en_sp_st 0.425851\n",
"en_sp_tt 0.412468\n",
"en_wr_st 0.434631\n",
"en_wr_tt 0.441960\n",
"it_sp_st 0.498941\n",
"it_sp_tt 0.459118\n",
"it_wr_st 0.499294\n",
"it_wr_tt 0.500667\n",
"Name: sttr, dtype: float64 \n",
"\n",
"Median TTR by Text Type:\n",
" text_type\n",
"en_sp_st 0.422000\n",
"en_sp_tt 0.409408\n",
"en_wr_st 0.434870\n",
"en_wr_tt 0.442000\n",
"it_sp_st 0.498000\n",
"it_sp_tt 0.465000\n",
"it_wr_st 0.503000\n",
"it_wr_tt 0.496000\n",
"Name: sttr, dtype: float64\n"
]
}
]
}
]
}