{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5eecbeeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm # Fortschrittsanzeige für pandas\n",
    "tqdm.pandas()\n",
    "tweet_csv = '../data/copbird_table_tweet.csv'\n",
    "entity_csv =  '../data/copbird_table_entity.csv'\n",
    "user_csv = '../data/copbird_table_user.csv'\n",
    "\n",
    "limit = None\n",
    "tweets = pd.read_csv(tweet_csv, nrows=limit)\n",
    "entities = pd.read_csv(entity_csv, nrows=limit)\n",
    "users = pd.read_csv(user_csv, nrows=limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1ad0f35a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "360008"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c0a49030",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/pandas/core/strings/accessor.py:101: UserWarning: This pattern has match groups. To actually get the groups, use str.extract.\n",
      "  return func(self, *args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>created_at</th>\n",
       "      <th>user_id</th>\n",
       "      <th>like_count</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>reply_count</th>\n",
       "      <th>quote_count</th>\n",
       "      <th>contains</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1321023114071969792</td>\n",
       "      <td>#Zeugengesucht\\nDie Hintergründe zu dem Tötung...</td>\n",
       "      <td>2020-10-27 09:37:08</td>\n",
       "      <td>2397974054</td>\n",
       "      <td>20.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1321025127388188673</td>\n",
       "      <td>RT @bka: EUROPE´S MOST WANTED – Sexualstraftät...</td>\n",
       "      <td>2020-10-27 09:45:08</td>\n",
       "      <td>2397974054</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id                                         tweet_text  \\\n",
       "1  1321023114071969792  #Zeugengesucht\\nDie Hintergründe zu dem Tötung...   \n",
       "2  1321025127388188673  RT @bka: EUROPE´S MOST WANTED – Sexualstraftät...   \n",
       "\n",
       "            created_at     user_id  like_count  retweet_count  reply_count  \\\n",
       "1  2020-10-27 09:37:08  2397974054        20.0           24.0          4.0   \n",
       "2  2020-10-27 09:45:08  2397974054         NaN            NaN          NaN   \n",
       "\n",
       "   quote_count  contains  \n",
       "1          1.0      True  \n",
       "2          NaN      True  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets[\"contains\"] = tweets[\"tweet_text\"].str.contains(r'(https://t.co/)')\n",
    "tweets_subset = tweets[tweets[\"contains\"]]\n",
    "tweets_link = tweets_subset\n",
    "tweets_link.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "a5e2d37e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-43-393ebe765dff>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tweets_subset[\"link\"] = tweets_subset[\"tweet_text\"].str.extract(r'(https://t.co/[\\w]*)[\\w$|\\s|\\n][^(.*).{1} ]')\n"
     ]
    }
   ],
   "source": [
    "tweets_subset[\"link\"] = tweets_subset[\"tweet_text\"].str.extract(r'(https://t.co/[\\w]*)[\\w$|\\s|\\n][^(.*).{1} ]')\n",
    "tryout = tweets_subset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "957a6b06",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tryout = tryout.head(9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b768104e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import requests\n",
    "response_arr = []\n",
    "#url = tryout[\"link\"].to_string(index=False)\n",
    "df = pd.DataFrame(data=response_arr, columns=[\"full_link\"])\n",
    "\n",
    "for i in tryout.index:\n",
    "    try:\n",
    "        response = requests.get(tryout[\"link\"][i])\n",
    "        response_arr.append(response.url)\n",
    "        \n",
    "    except:\n",
    "        response_arr.append(\"error\")\n",
    "    finally:\n",
    "        \n",
    "        df = pd.DataFrame([i, response.url], columns=[\"full_link\"])\n",
    "        df.to_csv('links.csv', mode='a', header=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1c5f3260",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'response_arr' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-30722b9a87cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresponse_arr_orig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtryout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"link_long\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"nan\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponse_arr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mtryout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"link_long\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'response_arr' is not defined"
     ]
    }
   ],
   "source": [
    "response_arr_orig = response_arr\n",
    "tryout[\"link_long\"]=float(\"nan\")\n",
    "index = 0\n",
    "for i in response_arr:\n",
    "    tryout[\"link_long\"][index]=i\n",
    "    index = index+1\n",
    "print(index)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}