373 lines
12 KiB
Plaintext
373 lines
12 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Chapter 9 - Data Science\n",
|
|
"## Data Preparation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 0 - Setting up the notebook"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import json\n",
|
|
"import random\n",
|
|
"from datetime import date, timedelta\n",
|
|
"\n",
|
|
"import faker"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 1 - Preparing the Data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# create the faker to populate the data\n",
|
|
"fake = faker.Faker()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"usernames = set()\n",
|
|
"usernames_no = 1000\n",
|
|
"\n",
|
|
"# populate the set with 1000 unique usernames\n",
|
|
"while len(usernames) < usernames_no:\n",
|
|
" usernames.add(fake.user_name())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"['{\"username\": \"susan42\", \"name\": \"Emily Smith\", \"gender\": \"F\", \"email\": \"vmckinney@leon.com\", \"age\": 53, \"address\": \"66537 Riley Mission Apt. 337\\\\nNorth Jennifer, NH 95781\"}',\n",
|
|
" '{\"username\": \"sarahcarpenter\", \"name\": \"Michael Kane\", \"gender\": \"M\", \"email\": \"tamara51@yahoo.com\", \"age\": 58, \"address\": \"7129 Patrick Walks Suite 215\\\\nLaurenside, LA 97179\"}',\n",
|
|
" '{\"username\": \"kevin37\", \"name\": \"Nathaniel Miller\", \"gender\": \"M\", \"email\": \"maria21@gmail.com\", \"age\": 36, \"address\": \"8247 Manning Burgs Suite 806\\\\nLopezshire, MS 06606\"}']"
|
|
]
|
|
},
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"def get_random_name_and_gender():\n",
|
|
" skew = .6 # 60% of users will be female\n",
|
|
" male = random.random() > skew\n",
|
|
" if male:\n",
|
|
" return fake.name_male(), 'M'\n",
|
|
" else:\n",
|
|
" return fake.name_female(), 'F'\n",
|
|
"\n",
|
|
"# for each username, create a complete user profile\n",
|
|
"# simulate user data coming from an API. It is a list\n",
|
|
"# of JSON strings (users).\n",
|
|
"def get_users(usernames):\n",
|
|
" users = []\n",
|
|
" for username in usernames:\n",
|
|
" name, gender = get_random_name_and_gender()\n",
|
|
" user = {\n",
|
|
" 'username': username,\n",
|
|
" 'name': name,\n",
|
|
" 'gender': gender,\n",
|
|
" 'email': fake.email(),\n",
|
|
" 'age': fake.random_int(min=18, max=90),\n",
|
|
" 'address': fake.address(),\n",
|
|
" }\n",
|
|
" users.append(json.dumps(user))\n",
|
|
" return users\n",
|
|
"\n",
|
|
"users = get_users(usernames)\n",
|
|
"users[:3]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# campaign name format:\n",
|
|
"# InternalType_StartDate_EndDate_TargetAge_TargetGender_Currency\n",
|
|
"def get_type():\n",
|
|
" # just some gibberish internal codes\n",
|
|
" types = ['AKX', 'BYU', 'GRZ', 'KTR']\n",
|
|
" return random.choice(types)\n",
|
|
"\n",
|
|
"def get_start_end_dates():\n",
|
|
" duration = random.randint(1, 2 * 365)\n",
|
|
" offset = random.randint(-365, 365)\n",
|
|
" start = date.today() - timedelta(days=offset)\n",
|
|
" end = start + timedelta(days=duration)\n",
|
|
" \n",
|
|
" def _format_date(date_):\n",
|
|
" return date_.strftime(\"%Y%m%d\")\n",
|
|
" \n",
|
|
" return _format_date(start), _format_date(end)\n",
|
|
"\n",
|
|
"def get_age():\n",
|
|
" age = random.randrange(20, 46, 5)\n",
|
|
" diff = random.randrange(5, 26, 5)\n",
|
|
" return '{}-{}'.format(age, age + diff)\n",
|
|
"\n",
|
|
"def get_gender():\n",
|
|
" return random.choice(('M', 'F', 'B'))\n",
|
|
"\n",
|
|
"def get_currency():\n",
|
|
" return random.choice(('GBP', 'EUR', 'USD'))\n",
|
|
"\n",
|
|
"def get_campaign_name():\n",
|
|
" separator = '_'\n",
|
|
" type_ = get_type()\n",
|
|
" start, end = get_start_end_dates()\n",
|
|
" age = get_age()\n",
|
|
" gender = get_gender()\n",
|
|
" currency = get_currency()\n",
|
|
" return separator.join(\n",
|
|
" (type_, start, end, age, gender, currency))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# campaign data:\n",
|
|
"# name, budget, spent, clicks, impressions\n",
|
|
"def get_campaign_data():\n",
|
|
" name = get_campaign_name()\n",
|
|
" budget = random.randint(10**3, 10**6)\n",
|
|
" spent = random.randint(10**2, budget) \n",
|
|
" clicks = int(random.triangular(10**2, 10**5, 0.2 * 10**5)) \n",
|
|
" impressions = int(random.gauss(0.5 * 10**6, 2))\n",
|
|
" return {\n",
|
|
" 'cmp_name': name,\n",
|
|
" 'cmp_bgt': budget,\n",
|
|
" 'cmp_spent': spent,\n",
|
|
" 'cmp_clicks': clicks,\n",
|
|
" 'cmp_impr': impressions\n",
|
|
" }"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# assemble the logic to get the final version of the rough data\n",
|
|
"# data will be a list of dictionaries. Each dictionary will follow\n",
|
|
"# this structure:\n",
|
|
"# {'user': user_json, 'campaigns': [c1, c2, ...]}\n",
|
|
"# where user_json is the JSON string version of a user data dict\n",
|
|
"# and c1, c2, ... are campaign dicts as returned by\n",
|
|
"# get_campaign_data\n",
|
|
"\n",
|
|
"def get_data(users):\n",
|
|
" data = []\n",
|
|
" for user in users:\n",
|
|
" campaigns = [get_campaign_data()\n",
|
|
" for _ in range(random.randint(2, 8))]\n",
|
|
" data.append({'user': user, 'campaigns': campaigns})\n",
|
|
" return data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 2 - Cleaning the data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[{'user': '{\"username\": \"susan42\", \"name\": \"Emily Smith\", \"gender\": \"F\", \"email\": \"vmckinney@leon.com\", \"age\": 53, \"address\": \"66537 Riley Mission Apt. 337\\\\nNorth Jennifer, NH 95781\"}',\n",
|
|
" 'campaigns': [{'cmp_name': 'GRZ_20210131_20210411_30-40_F_GBP',\n",
|
|
" 'cmp_bgt': 253951,\n",
|
|
" 'cmp_spent': 17953,\n",
|
|
" 'cmp_clicks': 52573,\n",
|
|
" 'cmp_impr': 500001},\n",
|
|
" {'cmp_name': 'BYU_20210109_20221204_30-35_M_GBP',\n",
|
|
" 'cmp_bgt': 150314,\n",
|
|
" 'cmp_spent': 125884,\n",
|
|
" 'cmp_clicks': 24575,\n",
|
|
" 'cmp_impr': 499999},\n",
|
|
" {'cmp_name': 'GRZ_20211124_20220921_20-35_B_EUR',\n",
|
|
" 'cmp_bgt': 791397,\n",
|
|
" 'cmp_spent': 480963,\n",
|
|
" 'cmp_clicks': 39668,\n",
|
|
" 'cmp_impr': 499999},\n",
|
|
" {'cmp_name': 'GRZ_20210727_20220211_35-45_B_EUR',\n",
|
|
" 'cmp_bgt': 910204,\n",
|
|
" 'cmp_spent': 339997,\n",
|
|
" 'cmp_clicks': 16698,\n",
|
|
" 'cmp_impr': 500000},\n",
|
|
" {'cmp_name': 'BYU_20220216_20220407_20-25_F_EUR',\n",
|
|
" 'cmp_bgt': 393134,\n",
|
|
" 'cmp_spent': 158930,\n",
|
|
" 'cmp_clicks': 46631,\n",
|
|
" 'cmp_impr': 500000}]},\n",
|
|
" {'user': '{\"username\": \"sarahcarpenter\", \"name\": \"Michael Kane\", \"gender\": \"M\", \"email\": \"tamara51@yahoo.com\", \"age\": 58, \"address\": \"7129 Patrick Walks Suite 215\\\\nLaurenside, LA 97179\"}',\n",
|
|
" 'campaigns': [{'cmp_name': 'BYU_20220324_20221230_20-45_B_USD',\n",
|
|
" 'cmp_bgt': 819948,\n",
|
|
" 'cmp_spent': 105178,\n",
|
|
" 'cmp_clicks': 27755,\n",
|
|
" 'cmp_impr': 500004},\n",
|
|
" {'cmp_name': 'GRZ_20201008_20210604_30-40_B_GBP',\n",
|
|
" 'cmp_bgt': 829698,\n",
|
|
" 'cmp_spent': 143193,\n",
|
|
" 'cmp_clicks': 88114,\n",
|
|
" 'cmp_impr': 499998},\n",
|
|
" {'cmp_name': 'GRZ_20210710_20211130_25-30_B_USD',\n",
|
|
" 'cmp_bgt': 815470,\n",
|
|
" 'cmp_spent': 79377,\n",
|
|
" 'cmp_clicks': 28283,\n",
|
|
" 'cmp_impr': 500002},\n",
|
|
" {'cmp_name': 'AKX_20211028_20220112_25-35_F_USD',\n",
|
|
" 'cmp_bgt': 944028,\n",
|
|
" 'cmp_spent': 657427,\n",
|
|
" 'cmp_clicks': 6668,\n",
|
|
" 'cmp_impr': 499999},\n",
|
|
" {'cmp_name': 'AKX_20211025_20220314_25-35_M_EUR',\n",
|
|
" 'cmp_bgt': 39136,\n",
|
|
" 'cmp_spent': 29326,\n",
|
|
" 'cmp_clicks': 20927,\n",
|
|
" 'cmp_impr': 499998},\n",
|
|
" {'cmp_name': 'BYU_20211227_20220615_20-35_F_USD',\n",
|
|
" 'cmp_bgt': 940412,\n",
|
|
" 'cmp_spent': 131757,\n",
|
|
" 'cmp_clicks': 57384,\n",
|
|
" 'cmp_impr': 500001},\n",
|
|
" {'cmp_name': 'AKX_20220323_20230602_35-55_M_GBP',\n",
|
|
" 'cmp_bgt': 545483,\n",
|
|
" 'cmp_spent': 96427,\n",
|
|
" 'cmp_clicks': 43290,\n",
|
|
" 'cmp_impr': 499999},\n",
|
|
" {'cmp_name': 'AKX_20210917_20220912_35-55_B_USD',\n",
|
|
" 'cmp_bgt': 129347,\n",
|
|
" 'cmp_spent': 4747,\n",
|
|
" 'cmp_clicks': 88217,\n",
|
|
" 'cmp_impr': 499999}]}]"
|
|
]
|
|
},
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# fetch simulated rough data\n",
|
|
"rough_data = get_data(users)\n",
|
|
"\n",
|
|
"rough_data[:2] # let's take a peek"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[{'cmp_name': 'GRZ_20210131_20210411_30-40_F_GBP',\n",
|
|
" 'cmp_bgt': 253951,\n",
|
|
" 'cmp_spent': 17953,\n",
|
|
" 'cmp_clicks': 52573,\n",
|
|
" 'cmp_impr': 500001,\n",
|
|
" 'user': '{\"username\": \"susan42\", \"name\": \"Emily Smith\", \"gender\": \"F\", \"email\": \"vmckinney@leon.com\", \"age\": 53, \"address\": \"66537 Riley Mission Apt. 337\\\\nNorth Jennifer, NH 95781\"}'},\n",
|
|
" {'cmp_name': 'BYU_20210109_20221204_30-35_M_GBP',\n",
|
|
" 'cmp_bgt': 150314,\n",
|
|
" 'cmp_spent': 125884,\n",
|
|
" 'cmp_clicks': 24575,\n",
|
|
" 'cmp_impr': 499999,\n",
|
|
" 'user': '{\"username\": \"susan42\", \"name\": \"Emily Smith\", \"gender\": \"F\", \"email\": \"vmckinney@leon.com\", \"age\": 53, \"address\": \"66537 Riley Mission Apt. 337\\\\nNorth Jennifer, NH 95781\"}'}]"
|
|
]
|
|
},
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Let's start from having a different version of the data\n",
|
|
"# I want a list whose items will be dicts. Each dict is \n",
|
|
"# the original campaign dict plus the user JSON\n",
|
|
"\n",
|
|
"data = []\n",
|
|
"for datum in rough_data:\n",
|
|
" for campaign in datum['campaigns']:\n",
|
|
" campaign.update({'user': datum['user']})\n",
|
|
" data.append(campaign)\n",
|
|
"data[:2] # let's take another peek"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Warning: Uncommenting and executing this cell will overwrite data.json\n",
|
|
"#with open('data.json', 'w') as stream:\n",
|
|
"# stream.write(json.dumps(data))"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|