diff --git a/.github/scripts/clean_kernelspecs.py b/.github/scripts/clean_kernelspecs.py index c6e5e85..02b2cd1 100644 --- a/.github/scripts/clean_kernelspecs.py +++ b/.github/scripts/clean_kernelspecs.py @@ -2,13 +2,20 @@ import glob for nb_path in glob.glob("**/*.ipynb", recursive=True): - with open(nb_path) as f: - nb = nbformat.read(f, as_version=4) + with open(nb_path, encoding = "utf-8") as f: + print(nb_path) + try: + nb = nbformat.read(f, as_version=4) + + except Exception as e: + print(e) + continue + nb['metadata']['kernelspec'] = { "name": "python3", "display_name": "Python 3", "language": "python" } - with open(nb_path, 'w') as f: + with open(nb_path, 'w', encoding = "utf-8") as f: nbformat.write(nb, f) diff --git a/practicals_jn_book/week_1/finalbook_part1.ipynb b/practicals_jn_book/week_1/finalbook_part1.ipynb index 54bda40..5e7d9d6 100644 --- a/practicals_jn_book/week_1/finalbook_part1.ipynb +++ b/practicals_jn_book/week_1/finalbook_part1.ipynb @@ -631,7 +631,7 @@ "\n", "### Assignment 5\n", "\n", - "- **First sort all the data by Totalkg score, make sure the Totalkg is on top of your DataFrame. Print out the DataFrame. What do you notice?**\n", + "- **First sort all the data by Totalkg score, make sure the lowest Totalkg is on top of your DataFrame. Print out the DataFrame. What do you notice?**\n", "\n", "- **Remove all rows that contain missing values. Print out the DataFrame. What happened to the index?**\n", "\n", diff --git a/practicals_jn_book/week_1/finalbook_part2.ipynb b/practicals_jn_book/week_1/finalbook_part2.ipynb index 213d340..e69de29 100644 --- a/practicals_jn_book/week_1/finalbook_part2.ipynb +++ b/practicals_jn_book/week_1/finalbook_part2.ipynb @@ -1,3371 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "tags": [ - "remove-cell" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "My Python version is: 3.13.12\n", - "My Numpy version is: 2.4.2\n", - "My Pandas version is: 3.0.1\n" - ] - } - ], - "source": [ - "import platform # imports go at the top of your script\n", - "import os # start with imports from the standard library followed by a space\n", - "\n", - "import numpy as np # then import custom/user packages\n", - "import pandas as pd # you can use shorter names for easy reference\n", - "import matplotlib.pyplot as plt # np, pd, and plt are common for these packages\n", - "\n", - "print(f\"My Python version is: {platform.python_version()}\")\n", - "print(f\"My Numpy version is: {np.__version__}\")\n", - "print(f\"My Pandas version is: {pd.__version__}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Pandas: working with data\n", - "\n", - "So that was quite the introduction, for the next section we will use part of the [OpenPowerlifting](https://www.openpowerlifting.org/faq) dataset. You wanted a lot of data so you will get a lot of data - this dataset has 1,343,621ish rows and 37 columns on results of international powerlifting competitions (in 2019, probably more now). That is a lot of data, even when put in a plain text format. You would have a bad time with traditional tools such as Excel, but this is where Pandas shines! \n", - "\n", - "```{note} \n", - "This notebook uses data from the OpenPowerlifting project, https://www.openpowerlifting.org.\n", - " You may download a copy of the data at https://gitlab.com/openpowerlifting/opl-data.\n", - "```\n", - "\n", - ">**Objective:**\n", - ">\n", - ">Your research objective is to see how the total lifted weight of lifters in the IPF in the world championships has developed over time, to answer this question we will use a real dataset.\n", - ">\n", - "\n", - "## Reading data\n", - "\n", - "If you work with real data, you preferably wouldn't construct the DataFrame from scratch like we did before. Pandas has a lot of ways to read data into memory and into a DataFrame. This includes, but is not limited to:\n", - "\n", - "- [read_csv](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html#pandas.read_csv)\n", - "- [read_clipboard](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_clipboard.html#pandas.read_clipboard)\n", - "- [read_excel](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_excel.html#pandas.read_excel)\n", - "- [read_json](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_json.html#pandas.read_json)\n", - "- [read_html](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_html.html#pandas.read_html)\n", - "- [read_hdf](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_hdf.html#pandas.read_hdf)\n", - "- [read_spss](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_spss.html#pandas.read_spss)\n", - "\n", - "You can pretty much coerce any file into a DataFrame with these methods. You will always need to parse some data yourself, but try to minimize that as much as reasonable (your implementation is probably not as fast as theirs, if it is, send them a pull request).\n", - "\n", - "### Assignment 6\n", - "\n", - "We prepared a subset of the powerlifting data on [here](https://raw.githubusercontent.com/Alek050/big_data_practicals/refs/heads/main/data/week_1/IPF_Worlds.csv). It only contains data from the World Championships for the International Powerlifting Federation.\n", - "\n", - "As this is a comma separated values file we need to import the data using the ``pd.read_csv()`` function. The read_csv function has a lot of optional keyword arguments. Remember to use IPython and Spyder (``ctrl+I``) to help you along!\n", - "\n", - "- **Load the sample dataset as df_ipf from [GitHub](https://github.com/Alek050/big_data_practicals/tree/main/data/week_1) or from the link above. Simply use ``read_csv()`` on the file path or on the link.**\n", - "\n", - "- **Print the DataFrame.**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameSexEventEquipmentAgeAgeClassBirthYearClassDivisionBodyweightKgWeightClassKgSquat1KgSquat2KgSquat3KgSquat4KgBest3SquatKgBench1KgBench2KgBench3KgBench4KgBest3BenchKgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgBest3DeadliftKgTotalKgPlaceWilksMcCullochGlossbrennerIPFPointsTestedCountryFederationParentFederationDateMeetCountryMeetStateMeetName
0Sergey FedosienkoMSBDRaw31.524-3424-39Open58.2059200.0215.0225.5NaN225.5150.0160.0165.0NaN165.0230.0255.0270.5NaN270.5661.01579.90579.90567.26870.42YesRussiaIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
1Dariusz WszołaMSBDRaw35.535-3924-39Open58.3059200.0210.0215.0NaN215.0142.5150.0152.5NaN152.5197.5205.0-210.0NaN205.0572.52501.45501.45490.47741.40YesPolandIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
2Franklin LeónMSBDRaw30.524-3424-39Open58.4559180.0192.5200.0NaN200.0130.0140.0-145.0NaN140.0210.0220.0-235.0NaN220.0560.03489.31489.31478.53721.77YesEcuadorIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
3Takaharu EbiharaMSBDRaw33.524-3424-39Open58.7059165.0180.0185.0NaN185.0130.0140.0-142.5NaN140.0200.0210.0215.0NaN215.0540.04469.96469.96459.49690.42YesJapanIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
4Mohamed LakehalMSBDRaw31.024-3424-39Open58.5059195.0210.0-215.0NaN210.0105.0110.0-112.5NaN110.0200.0215.0-220.0NaN215.0535.05467.10467.10456.78685.25YesAlgeriaIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
........................................................................................................................
4657Ina KoolhaasFSBDRaw71.570-7470-999Masters 460.2063-85.0-85.085.0NaN85.0-52.552.5-55.0NaN52.5120.0127.5133.5NaN133.5271.01301.36506.58266.25474.80YesNetherlandsIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4658Tatyana FominaFSBDRaw72.570-7470-999Masters 461.606380.085.087.5NaN87.542.545.0-47.5NaN45.0105.0110.0115.0NaN115.0247.52270.39464.54238.83427.58YesRussiaIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4659Anne Mari ClausenFSBDRaw76.575-7970-999Masters 471.607260.065.070.0NaN70.042.5-47.5-47.5NaN42.580.087.592.5NaN92.5205.01200.83376.76176.93332.63YesNorwayIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4660Shirley WebbFSBDRaw81.580-99970-999Masters 494.3584+30.0-35.040.0NaN40.025.035.0-37.5NaN35.080.0100.0-120.0NaN100.0175.01148.48311.21128.44263.38YesUSAIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4661Margaret AlbertFSBDRaw72.570-7470-999Masters 4104.4584+57.562.5-65.0NaN62.545.0-50.0-50.0NaN45.0-95.0NaNNaNNaNNaNNaNDQNaNNaNNaNNaNYesUSAIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
\n", - "

4662 rows × 39 columns

\n", - "
" - ], - "text/plain": [ - " Name Sex Event Equipment Age AgeClass BirthYearClass \\\n", - "0 Sergey Fedosienko M SBD Raw 31.5 24-34 24-39 \n", - "1 Dariusz Wszoła M SBD Raw 35.5 35-39 24-39 \n", - "2 Franklin León M SBD Raw 30.5 24-34 24-39 \n", - "3 Takaharu Ebihara M SBD Raw 33.5 24-34 24-39 \n", - "4 Mohamed Lakehal M SBD Raw 31.0 24-34 24-39 \n", - "... ... .. ... ... ... ... ... \n", - "4657 Ina Koolhaas F SBD Raw 71.5 70-74 70-999 \n", - "4658 Tatyana Fomina F SBD Raw 72.5 70-74 70-999 \n", - "4659 Anne Mari Clausen F SBD Raw 76.5 75-79 70-999 \n", - "4660 Shirley Webb F SBD Raw 81.5 80-999 70-999 \n", - "4661 Margaret Albert F SBD Raw 72.5 70-74 70-999 \n", - "\n", - " Division BodyweightKg WeightClassKg Squat1Kg Squat2Kg Squat3Kg \\\n", - "0 Open 58.20 59 200.0 215.0 225.5 \n", - "1 Open 58.30 59 200.0 210.0 215.0 \n", - "2 Open 58.45 59 180.0 192.5 200.0 \n", - "3 Open 58.70 59 165.0 180.0 185.0 \n", - "4 Open 58.50 59 195.0 210.0 -215.0 \n", - "... ... ... ... ... ... ... \n", - "4657 Masters 4 60.20 63 -85.0 -85.0 85.0 \n", - "4658 Masters 4 61.60 63 80.0 85.0 87.5 \n", - "4659 Masters 4 71.60 72 60.0 65.0 70.0 \n", - "4660 Masters 4 94.35 84+ 30.0 -35.0 40.0 \n", - "4661 Masters 4 104.45 84+ 57.5 62.5 -65.0 \n", - "\n", - " Squat4Kg Best3SquatKg Bench1Kg Bench2Kg Bench3Kg Bench4Kg \\\n", - "0 NaN 225.5 150.0 160.0 165.0 NaN \n", - "1 NaN 215.0 142.5 150.0 152.5 NaN \n", - "2 NaN 200.0 130.0 140.0 -145.0 NaN \n", - "3 NaN 185.0 130.0 140.0 -142.5 NaN \n", - "4 NaN 210.0 105.0 110.0 -112.5 NaN \n", - "... ... ... ... ... ... ... \n", - "4657 NaN 85.0 -52.5 52.5 -55.0 NaN \n", - "4658 NaN 87.5 42.5 45.0 -47.5 NaN \n", - "4659 NaN 70.0 42.5 -47.5 -47.5 NaN \n", - "4660 NaN 40.0 25.0 35.0 -37.5 NaN \n", - "4661 NaN 62.5 45.0 -50.0 -50.0 NaN \n", - "\n", - " Best3BenchKg Deadlift1Kg Deadlift2Kg Deadlift3Kg Deadlift4Kg \\\n", - "0 165.0 230.0 255.0 270.5 NaN \n", - "1 152.5 197.5 205.0 -210.0 NaN \n", - "2 140.0 210.0 220.0 -235.0 NaN \n", - "3 140.0 200.0 210.0 215.0 NaN \n", - "4 110.0 200.0 215.0 -220.0 NaN \n", - "... ... ... ... ... ... \n", - "4657 52.5 120.0 127.5 133.5 NaN \n", - "4658 45.0 105.0 110.0 115.0 NaN \n", - "4659 42.5 80.0 87.5 92.5 NaN \n", - "4660 35.0 80.0 100.0 -120.0 NaN \n", - "4661 45.0 -95.0 NaN NaN NaN \n", - "\n", - " Best3DeadliftKg TotalKg Place Wilks McCulloch Glossbrenner \\\n", - "0 270.5 661.0 1 579.90 579.90 567.26 \n", - "1 205.0 572.5 2 501.45 501.45 490.47 \n", - "2 220.0 560.0 3 489.31 489.31 478.53 \n", - "3 215.0 540.0 4 469.96 469.96 459.49 \n", - "4 215.0 535.0 5 467.10 467.10 456.78 \n", - "... ... ... ... ... ... ... \n", - "4657 133.5 271.0 1 301.36 506.58 266.25 \n", - "4658 115.0 247.5 2 270.39 464.54 238.83 \n", - "4659 92.5 205.0 1 200.83 376.76 176.93 \n", - "4660 100.0 175.0 1 148.48 311.21 128.44 \n", - "4661 NaN NaN DQ NaN NaN NaN \n", - "\n", - " IPFPoints Tested Country Federation ParentFederation Date \\\n", - "0 870.42 Yes Russia IPF IPF 2014-06-01 \n", - "1 741.40 Yes Poland IPF IPF 2014-06-01 \n", - "2 721.77 Yes Ecuador IPF IPF 2014-06-01 \n", - "3 690.42 Yes Japan IPF IPF 2014-06-01 \n", - "4 685.25 Yes Algeria IPF IPF 2014-06-01 \n", - "... ... ... ... ... ... ... \n", - "4657 474.80 Yes Netherlands IPF IPF 2019-06-04 \n", - "4658 427.58 Yes Russia IPF IPF 2019-06-04 \n", - "4659 332.63 Yes Norway IPF IPF 2019-06-04 \n", - "4660 263.38 Yes USA IPF IPF 2019-06-04 \n", - "4661 NaN Yes USA IPF IPF 2019-06-04 \n", - "\n", - " MeetCountry MeetState MeetName \n", - "0 South Africa NaN World Classic Powerlifting Championships \n", - "1 South Africa NaN World Classic Powerlifting Championships \n", - "2 South Africa NaN World Classic Powerlifting Championships \n", - "3 South Africa NaN World Classic Powerlifting Championships \n", - "4 South Africa NaN World Classic Powerlifting Championships \n", - "... ... ... ... \n", - "4657 Sweden NaN World Classic Powerlifting Championships \n", - "4658 Sweden NaN World Classic Powerlifting Championships \n", - "4659 Sweden NaN World Classic Powerlifting Championships \n", - "4660 Sweden NaN World Classic Powerlifting Championships \n", - "4661 Sweden NaN World Classic Powerlifting Championships \n", - "\n", - "[4662 rows x 39 columns]" - ] - }, - "execution_count": 61, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# If you have the file locally on you device:\n", - "df_ipf = pd.read_csv(\"IPF_Worlds.csv\")\n", - "# Or ownload it from the web (if it is publicaly available):\n", - "# df_ipf = pd.read_csv(\"https://raw.githubusercontent.com/Alek050/big_data_practicals/refs/heads/main/data/week_1/IPF_Worlds.csv\")\n", - "df_ipf" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inspecting the data\n", - "\n", - "Now it is time to inspect what we are working with. The dataset is also used for the rankings on [openpowerlifting.org](https://www.openpowerlifting.org/). You can use that website to validate what you find in your data. \n", - "It is usually a good idea to spend some time on getting familiar with your data. Familiarizing with your data can prevent you from making mistakes later on and making false (statistical) conclusions.\n", - "\n", - "The first step is always to just look at the data. ``print()``, like we did before is not going to cut it now as the data are way too large to display in console. There are a couple of DataFrame methods that are more suitable for this, namely:\n", - "- df.head(): to view the first few rows\n", - "- df.tail(): to view the last few rows\n", - "- df.columns(): to view the columns we have\n", - "- df.describe(): to give a quantitative description of the data\n", - "- df.info(): to get information on datatypes and memory usage\n", - "\n", - "### Assignment 7\n", - "\n", - "How much memory are we using? What columns do we have? What datatypes do they contain? You can find out with a very simple command.\n", - "\n", - "- **Please use a method on your DataFrame to find out how much memory the DataFrame uses and print it to the console.**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "RangeIndex: 4662 entries, 0 to 4661\n", - "Data columns (total 39 columns):\n", - " # Column Non-Null Count Dtype \n", - "--- ------ -------------- ----- \n", - " 0 Name 4662 non-null str \n", - " 1 Sex 4662 non-null str \n", - " 2 Event 4662 non-null str \n", - " 3 Equipment 4662 non-null str \n", - " 4 Age 4662 non-null float64\n", - " 5 AgeClass 4662 non-null str \n", - " 6 BirthYearClass 4662 non-null str \n", - " 7 Division 4662 non-null str \n", - " 8 BodyweightKg 4662 non-null float64\n", - " 9 WeightClassKg 4662 non-null str \n", - " 10 Squat1Kg 4661 non-null float64\n", - " 11 Squat2Kg 4643 non-null float64\n", - " 12 Squat3Kg 4575 non-null float64\n", - " 13 Squat4Kg 0 non-null float64\n", - " 14 Best3SquatKg 4603 non-null float64\n", - " 15 Bench1Kg 4651 non-null float64\n", - " 16 Bench2Kg 4634 non-null float64\n", - " 17 Bench3Kg 4566 non-null float64\n", - " 18 Bench4Kg 0 non-null float64\n", - " 19 Best3BenchKg 4623 non-null float64\n", - " 20 Deadlift1Kg 4641 non-null float64\n", - " 21 Deadlift2Kg 4595 non-null float64\n", - " 22 Deadlift3Kg 4498 non-null float64\n", - " 23 Deadlift4Kg 0 non-null float64\n", - " 24 Best3DeadliftKg 4590 non-null float64\n", - " 25 TotalKg 4503 non-null float64\n", - " 26 Place 4662 non-null str \n", - " 27 Wilks 4503 non-null float64\n", - " 28 McCulloch 4503 non-null float64\n", - " 29 Glossbrenner 4503 non-null float64\n", - " 30 IPFPoints 4501 non-null float64\n", - " 31 Tested 4662 non-null str \n", - " 32 Country 4662 non-null str \n", - " 33 Federation 4662 non-null str \n", - " 34 ParentFederation 4662 non-null str \n", - " 35 Date 4662 non-null str \n", - " 36 MeetCountry 4662 non-null str \n", - " 37 MeetState 690 non-null str \n", - " 38 MeetName 4662 non-null str \n", - "dtypes: float64(22), str(17)\n", - "memory usage: 1.9 MB\n" - ] - } - ], - "source": [ - "df_ipf.info()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you can see we are using a lot of memory. We will deal with this a little bit later on after we get a feeling of what is in the DataFrame. \n", - "\n", - "### Assignment 8\n", - "\n", - "Visual inspection can be done through fancy graphs, but you could take another step back and just look at the DataFrame itself. One of the methods listed above can be used to inspect the dataframe, please do so.\n", - "\n", - "- **Print the first three rows of the DataFrame. What do you notice?**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameSexEventEquipmentAgeAgeClassBirthYearClassDivisionBodyweightKgWeightClassKgSquat1KgSquat2KgSquat3KgSquat4KgBest3SquatKgBench1KgBench2KgBench3KgBench4KgBest3BenchKgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgBest3DeadliftKgTotalKgPlaceWilksMcCullochGlossbrennerIPFPointsTestedCountryFederationParentFederationDateMeetCountryMeetStateMeetName
0Sergey FedosienkoMSBDRaw31.524-3424-39Open58.2059200.0215.0225.5NaN225.5150.0160.0165.0NaN165.0230.0255.0270.5NaN270.5661.01579.90579.90567.26870.42YesRussiaIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
1Dariusz WszołaMSBDRaw35.535-3924-39Open58.3059200.0210.0215.0NaN215.0142.5150.0152.5NaN152.5197.5205.0-210.0NaN205.0572.52501.45501.45490.47741.40YesPolandIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
2Franklin LeónMSBDRaw30.524-3424-39Open58.4559180.0192.5200.0NaN200.0130.0140.0-145.0NaN140.0210.0220.0-235.0NaN220.0560.03489.31489.31478.53721.77YesEcuadorIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
\n", - "
" - ], - "text/plain": [ - " Name Sex Event Equipment Age AgeClass BirthYearClass \\\n", - "0 Sergey Fedosienko M SBD Raw 31.5 24-34 24-39 \n", - "1 Dariusz Wszoła M SBD Raw 35.5 35-39 24-39 \n", - "2 Franklin León M SBD Raw 30.5 24-34 24-39 \n", - "\n", - " Division BodyweightKg WeightClassKg Squat1Kg Squat2Kg Squat3Kg \\\n", - "0 Open 58.20 59 200.0 215.0 225.5 \n", - "1 Open 58.30 59 200.0 210.0 215.0 \n", - "2 Open 58.45 59 180.0 192.5 200.0 \n", - "\n", - " Squat4Kg Best3SquatKg Bench1Kg Bench2Kg Bench3Kg Bench4Kg \\\n", - "0 NaN 225.5 150.0 160.0 165.0 NaN \n", - "1 NaN 215.0 142.5 150.0 152.5 NaN \n", - "2 NaN 200.0 130.0 140.0 -145.0 NaN \n", - "\n", - " Best3BenchKg Deadlift1Kg Deadlift2Kg Deadlift3Kg Deadlift4Kg \\\n", - "0 165.0 230.0 255.0 270.5 NaN \n", - "1 152.5 197.5 205.0 -210.0 NaN \n", - "2 140.0 210.0 220.0 -235.0 NaN \n", - "\n", - " Best3DeadliftKg TotalKg Place Wilks McCulloch Glossbrenner IPFPoints \\\n", - "0 270.5 661.0 1 579.90 579.90 567.26 870.42 \n", - "1 205.0 572.5 2 501.45 501.45 490.47 741.40 \n", - "2 220.0 560.0 3 489.31 489.31 478.53 721.77 \n", - "\n", - " Tested Country Federation ParentFederation Date MeetCountry \\\n", - "0 Yes Russia IPF IPF 2014-06-01 South Africa \n", - "1 Yes Poland IPF IPF 2014-06-01 South Africa \n", - "2 Yes Ecuador IPF IPF 2014-06-01 South Africa \n", - "\n", - " MeetState MeetName \n", - "0 NaN World Classic Powerlifting Championships \n", - "1 NaN World Classic Powerlifting Championships \n", - "2 NaN World Classic Powerlifting Championships " - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_ipf.head(3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You probably did not get to see all columns of your DataFrame which is unfortunate. Luckily, there is an [option](https://pandas.pydata.org/pandas-docs/stable/user_guide/options.html) that let's you display more columns. \n", - "\n", - "- **Look up the documentation for pd.set_option in Spyder and set the maximum columns to 50. Now inspect the head of the DataFrame again.**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameSexEventEquipmentAgeAgeClassBirthYearClassDivisionBodyweightKgWeightClassKgSquat1KgSquat2KgSquat3KgSquat4KgBest3SquatKgBench1KgBench2KgBench3KgBench4KgBest3BenchKgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgBest3DeadliftKgTotalKgPlaceWilksMcCullochGlossbrennerIPFPointsTestedCountryFederationParentFederationDateMeetCountryMeetStateMeetName
0Sergey FedosienkoMSBDRaw31.524-3424-39Open58.2059200.0215.0225.5NaN225.5150.0160.0165.0NaN165.0230.0255.0270.5NaN270.5661.01579.90579.90567.26870.42YesRussiaIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
1Dariusz WszołaMSBDRaw35.535-3924-39Open58.3059200.0210.0215.0NaN215.0142.5150.0152.5NaN152.5197.5205.0-210.0NaN205.0572.52501.45501.45490.47741.40YesPolandIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
2Franklin LeónMSBDRaw30.524-3424-39Open58.4559180.0192.5200.0NaN200.0130.0140.0-145.0NaN140.0210.0220.0-235.0NaN220.0560.03489.31489.31478.53721.77YesEcuadorIPFIPF2014-06-01South AfricaNaNWorld Classic Powerlifting Championships
\n", - "
" - ], - "text/plain": [ - " Name Sex Event Equipment Age AgeClass BirthYearClass \\\n", - "0 Sergey Fedosienko M SBD Raw 31.5 24-34 24-39 \n", - "1 Dariusz Wszoła M SBD Raw 35.5 35-39 24-39 \n", - "2 Franklin León M SBD Raw 30.5 24-34 24-39 \n", - "\n", - " Division BodyweightKg WeightClassKg Squat1Kg Squat2Kg Squat3Kg \\\n", - "0 Open 58.20 59 200.0 215.0 225.5 \n", - "1 Open 58.30 59 200.0 210.0 215.0 \n", - "2 Open 58.45 59 180.0 192.5 200.0 \n", - "\n", - " Squat4Kg Best3SquatKg Bench1Kg Bench2Kg Bench3Kg Bench4Kg \\\n", - "0 NaN 225.5 150.0 160.0 165.0 NaN \n", - "1 NaN 215.0 142.5 150.0 152.5 NaN \n", - "2 NaN 200.0 130.0 140.0 -145.0 NaN \n", - "\n", - " Best3BenchKg Deadlift1Kg Deadlift2Kg Deadlift3Kg Deadlift4Kg \\\n", - "0 165.0 230.0 255.0 270.5 NaN \n", - "1 152.5 197.5 205.0 -210.0 NaN \n", - "2 140.0 210.0 220.0 -235.0 NaN \n", - "\n", - " Best3DeadliftKg TotalKg Place Wilks McCulloch Glossbrenner IPFPoints \\\n", - "0 270.5 661.0 1 579.90 579.90 567.26 870.42 \n", - "1 205.0 572.5 2 501.45 501.45 490.47 741.40 \n", - "2 220.0 560.0 3 489.31 489.31 478.53 721.77 \n", - "\n", - " Tested Country Federation ParentFederation Date MeetCountry \\\n", - "0 Yes Russia IPF IPF 2014-06-01 South Africa \n", - "1 Yes Poland IPF IPF 2014-06-01 South Africa \n", - "2 Yes Ecuador IPF IPF 2014-06-01 South Africa \n", - "\n", - " MeetState MeetName \n", - "0 NaN World Classic Powerlifting Championships \n", - "1 NaN World Classic Powerlifting Championships \n", - "2 NaN World Classic Powerlifting Championships " - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pd.set_option('display.max_columns', 50)\n", - "df_ipf.head(3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "What we get is pretty interesting. Powerlifting has three tries per lift, yet we see four tries per lift. This is probably because in some exceptions people get an extra lift (for example when a spotter accidentally touches the weight). If we inspect the column, we see that it contains a lot of missing values, ergo, not many people get a fourth lift. This is in line with what you would expect.\n", - "\n", - "Secondly, we see some negative values. Lifting negative weight should be easy. However, it seems more likely that these negative values are failed attempts. Pretty elegant way of including it in the dataset without using a lot of extra memory!\n", - "\n", - "- **Now print the last five values of the DataFrame**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameSexEventEquipmentAgeAgeClassBirthYearClassDivisionBodyweightKgWeightClassKgSquat1KgSquat2KgSquat3KgSquat4KgBest3SquatKgBench1KgBench2KgBench3KgBench4KgBest3BenchKgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgBest3DeadliftKgTotalKgPlaceWilksMcCullochGlossbrennerIPFPointsTestedCountryFederationParentFederationDateMeetCountryMeetStateMeetName
4657Ina KoolhaasFSBDRaw71.570-7470-999Masters 460.2063-85.0-85.085.0NaN85.0-52.552.5-55.0NaN52.5120.0127.5133.5NaN133.5271.01301.36506.58266.25474.80YesNetherlandsIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4658Tatyana FominaFSBDRaw72.570-7470-999Masters 461.606380.085.087.5NaN87.542.545.0-47.5NaN45.0105.0110.0115.0NaN115.0247.52270.39464.54238.83427.58YesRussiaIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4659Anne Mari ClausenFSBDRaw76.575-7970-999Masters 471.607260.065.070.0NaN70.042.5-47.5-47.5NaN42.580.087.592.5NaN92.5205.01200.83376.76176.93332.63YesNorwayIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4660Shirley WebbFSBDRaw81.580-99970-999Masters 494.3584+30.0-35.040.0NaN40.025.035.0-37.5NaN35.080.0100.0-120.0NaN100.0175.01148.48311.21128.44263.38YesUSAIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
4661Margaret AlbertFSBDRaw72.570-7470-999Masters 4104.4584+57.562.5-65.0NaN62.545.0-50.0-50.0NaN45.0-95.0NaNNaNNaNNaNNaNDQNaNNaNNaNNaNYesUSAIPFIPF2019-06-04SwedenNaNWorld Classic Powerlifting Championships
\n", - "
" - ], - "text/plain": [ - " Name Sex Event Equipment Age AgeClass BirthYearClass \\\n", - "4657 Ina Koolhaas F SBD Raw 71.5 70-74 70-999 \n", - "4658 Tatyana Fomina F SBD Raw 72.5 70-74 70-999 \n", - "4659 Anne Mari Clausen F SBD Raw 76.5 75-79 70-999 \n", - "4660 Shirley Webb F SBD Raw 81.5 80-999 70-999 \n", - "4661 Margaret Albert F SBD Raw 72.5 70-74 70-999 \n", - "\n", - " Division BodyweightKg WeightClassKg Squat1Kg Squat2Kg Squat3Kg \\\n", - "4657 Masters 4 60.20 63 -85.0 -85.0 85.0 \n", - "4658 Masters 4 61.60 63 80.0 85.0 87.5 \n", - "4659 Masters 4 71.60 72 60.0 65.0 70.0 \n", - "4660 Masters 4 94.35 84+ 30.0 -35.0 40.0 \n", - "4661 Masters 4 104.45 84+ 57.5 62.5 -65.0 \n", - "\n", - " Squat4Kg Best3SquatKg Bench1Kg Bench2Kg Bench3Kg Bench4Kg \\\n", - "4657 NaN 85.0 -52.5 52.5 -55.0 NaN \n", - "4658 NaN 87.5 42.5 45.0 -47.5 NaN \n", - "4659 NaN 70.0 42.5 -47.5 -47.5 NaN \n", - "4660 NaN 40.0 25.0 35.0 -37.5 NaN \n", - "4661 NaN 62.5 45.0 -50.0 -50.0 NaN \n", - "\n", - " Best3BenchKg Deadlift1Kg Deadlift2Kg Deadlift3Kg Deadlift4Kg \\\n", - "4657 52.5 120.0 127.5 133.5 NaN \n", - "4658 45.0 105.0 110.0 115.0 NaN \n", - "4659 42.5 80.0 87.5 92.5 NaN \n", - "4660 35.0 80.0 100.0 -120.0 NaN \n", - "4661 45.0 -95.0 NaN NaN NaN \n", - "\n", - " Best3DeadliftKg TotalKg Place Wilks McCulloch Glossbrenner \\\n", - "4657 133.5 271.0 1 301.36 506.58 266.25 \n", - "4658 115.0 247.5 2 270.39 464.54 238.83 \n", - "4659 92.5 205.0 1 200.83 376.76 176.93 \n", - "4660 100.0 175.0 1 148.48 311.21 128.44 \n", - "4661 NaN NaN DQ NaN NaN NaN \n", - "\n", - " IPFPoints Tested Country Federation ParentFederation Date \\\n", - "4657 474.80 Yes Netherlands IPF IPF 2019-06-04 \n", - "4658 427.58 Yes Russia IPF IPF 2019-06-04 \n", - "4659 332.63 Yes Norway IPF IPF 2019-06-04 \n", - "4660 263.38 Yes USA IPF IPF 2019-06-04 \n", - "4661 NaN Yes USA IPF IPF 2019-06-04 \n", - "\n", - " MeetCountry MeetState MeetName \n", - "4657 Sweden NaN World Classic Powerlifting Championships \n", - "4658 Sweden NaN World Classic Powerlifting Championships \n", - "4659 Sweden NaN World Classic Powerlifting Championships \n", - "4660 Sweden NaN World Classic Powerlifting Championships \n", - "4661 Sweden NaN World Classic Powerlifting Championships " - ] - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_ipf.tail(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Assignment 9\n", - "\n", - "You can get information on a column in a DataFrame (Series) with these methods:\n", - "\n", - "- unique: get the unique values from the Series\n", - "- nunique: get the number of unique values from the Series\n", - "- value_counts: get the, well, value counts of the Series\n", - "\n", - "- **Try all these methods on the Division column and print the outcomes. Remember how to use f-strings? Please use them to increase readability where needed.**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": { - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "[ 'Open', 'Sub-Juniors', 'Juniors', 'Masters 1', 'Masters 2',\n", - " 'Masters 3', 'Masters 4']\n", - "Length: 7, dtype: str \n", - "\n", - "There are : 7 divisions.\n", - "\n", - "Division\n", - "Open 1382\n", - "Juniors 974\n", - "Masters 1 680\n", - "Sub-Juniors 626\n", - "Masters 2 517\n", - "Masters 3 326\n", - "Masters 4 157\n", - "Name: count, dtype: int64\n" - ] - } - ], - "source": [ - "print(df_ipf[\"Division\"].unique(), \"\\n\") # \\n gives an extra new line after print, increases readability\n", - "print(f'There are : {df_ipf[\"Division\"].nunique()} divisions.\\n')\n", - "print(df_ipf[\"Division\"].value_counts())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Boolean indexing\n", - "\n", - "We will need to narrow our scope a bit. For starters, we don't need all the columns (OK we do but for educational purposes we don't). We also don't need all the rows:\n", - "\n", - "For the next part we will need [boolean](https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#boolean-indexing) array. Pandas (and NumPy, on which Pandas is built) does not use ``and``, ``or``, or ``not``. Instead, it uses ``&``, ``|``, and ``~``, respectively, which are normal, bona fide Python bitwise operators. For example, if we want data for female lifters that are over 80 kg only we can use:\n", - "\n", - "```Python\n", - "females_80 = df_ipf[(df_ipf[\"Sex\"] == \"F\") & \n", - " (df_ipf[\"BodyWeightKg\"] > 80)]\n", - "```\n", - "\n", - "Be sure to use parentheses when chaining conditions with ``& | ~`` or Pandas will throw an error. This is a very powerful way to select interesting data. Note that, again, you will probably get a view and not a copy.\n", - "\n", - "### Assignment 10\n", - "\n", - "To narrow our scope, we will have to drop some columns. You can either slice and select the columns you want or drop the columns with the ``.drop`` [\\[docs\\]](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.drop.html) method (the list of included columns is shorter but I want you to use drop 😬). \n", - "\n", - "```python\n", - "to_drop = ['Best3SquatKg', 'Best3BenchKg', 'Best3DeadliftKg', 'Wilks', 'McCulloch', 'TotalKg', 'Event',\n", - " 'Tested', 'AgeClass', 'Country', 'Glossbrenner', 'IPFPoints', 'MeetState', 'Place', 'Federation',\n", - " 'ParentFederation', 'MeetCountry', 'MeetName']\n", - "```\n", - "\n", - "- **Drop the abovementioned columns (look at the docs how to drop the columns).**\n", - "\n", - "- **Make sure the DataFrame only contains data from the \"Open\" division.**\n", - "\n", - "- **Choose a method to check whether the division only contains \"Open\".**\n", - "\n", - "- **Did you see what happened to the index? Reset and drop the index and print the first 5 rows.**\n", - "\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Division\n", - "Open 1382\n", - "Name: count, dtype: int64\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameSexEquipmentAgeBirthYearClassDivisionBodyweightKgWeightClassKgSquat1KgSquat2KgSquat3KgSquat4KgBench1KgBench2KgBench3KgBench4KgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgDate
0Sergey FedosienkoMRaw31.524-39Open58.2059200.0215.0225.5NaN150.0160.0165.0NaN230.0255.0270.5NaN2014-06-01
1Dariusz WszołaMRaw35.524-39Open58.3059200.0210.0215.0NaN142.5150.0152.5NaN197.5205.0-210.0NaN2014-06-01
2Franklin LeónMRaw30.524-39Open58.4559180.0192.5200.0NaN130.0140.0-145.0NaN210.0220.0-235.0NaN2014-06-01
3Takaharu EbiharaMRaw33.524-39Open58.7059165.0180.0185.0NaN130.0140.0-142.5NaN200.0210.0215.0NaN2014-06-01
4Mohamed LakehalMRaw31.024-39Open58.5059195.0210.0-215.0NaN105.0110.0-112.5NaN200.0215.0-220.0NaN2014-06-01
\n", - "
" - ], - "text/plain": [ - " Name Sex Equipment Age BirthYearClass Division \\\n", - "0 Sergey Fedosienko M Raw 31.5 24-39 Open \n", - "1 Dariusz Wszoła M Raw 35.5 24-39 Open \n", - "2 Franklin León M Raw 30.5 24-39 Open \n", - "3 Takaharu Ebihara M Raw 33.5 24-39 Open \n", - "4 Mohamed Lakehal M Raw 31.0 24-39 Open \n", - "\n", - " BodyweightKg WeightClassKg Squat1Kg Squat2Kg Squat3Kg Squat4Kg \\\n", - "0 58.20 59 200.0 215.0 225.5 NaN \n", - "1 58.30 59 200.0 210.0 215.0 NaN \n", - "2 58.45 59 180.0 192.5 200.0 NaN \n", - "3 58.70 59 165.0 180.0 185.0 NaN \n", - "4 58.50 59 195.0 210.0 -215.0 NaN \n", - "\n", - " Bench1Kg Bench2Kg Bench3Kg Bench4Kg Deadlift1Kg Deadlift2Kg \\\n", - "0 150.0 160.0 165.0 NaN 230.0 255.0 \n", - "1 142.5 150.0 152.5 NaN 197.5 205.0 \n", - "2 130.0 140.0 -145.0 NaN 210.0 220.0 \n", - "3 130.0 140.0 -142.5 NaN 200.0 210.0 \n", - "4 105.0 110.0 -112.5 NaN 200.0 215.0 \n", - "\n", - " Deadlift3Kg Deadlift4Kg Date \n", - "0 270.5 NaN 2014-06-01 \n", - "1 -210.0 NaN 2014-06-01 \n", - "2 -235.0 NaN 2014-06-01 \n", - "3 215.0 NaN 2014-06-01 \n", - "4 -220.0 NaN 2014-06-01 " - ] - }, - "execution_count": 67, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "to_drop = ['Best3SquatKg', 'Best3BenchKg', 'Best3DeadliftKg', 'Wilks', 'McCulloch', 'TotalKg', 'Event',\n", - " 'Tested', 'AgeClass', 'Country', 'Glossbrenner', 'IPFPoints', 'MeetState', 'Place', 'Federation',\n", - " 'ParentFederation', 'MeetCountry', 'MeetName']\n", - "\n", - "df_ipf.drop(to_drop, axis=1, inplace=True)\n", - "df_ipf = df_ipf[(df_ipf[\"Division\"] == \"Open\")]\n", - "\n", - "print(df_ipf.Division.value_counts())\n", - "\n", - "df_ipf = df_ipf.reset_index(drop=True)\n", - "df_ipf.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Simple descriptives\n", - "\n", - "Pandas also comes with a number of useful methods to describe your DataFrame. These methods return the min/max/std/... of the values over the requested axis. These methods help you understand your data.\n", - "\n", - "|Function| \tDescription|\n", - "|:---------|:----------|\n", - "|count()| \tNumber of non-null observations|\n", - "|sum()| \tSum of values|\n", - "|mean()| \tMean of Values|\n", - "|median()| \tMedian of Values|\n", - "|mode()| \tMode of values|\n", - "|std()| \tStandard Deviation of the Values|\n", - "|min()| \tMinimum Value|\n", - "|max()| \tMaximum Value|\n", - "|abs()| \tAbsolute Value|\n", - "|prod()| \tProduct of Values|\n", - "|cumsum()| \tCumulative Sum|\n", - "|cumprod()|\tCumulative Product|\n", - "|quantile()|Value at Quantile|\n", - "\n", - "### Assignment 11\n", - "\n", - "To see how strenght develops over the years we need to know how the maximum lifts developed over time. Step one is getting the maximum lifted weight.\n", - "\n", - "- **Calculate the mean, min, and maximum values for the ``[\"Squat1Kg\", \"Squat2Kg\", \"Squat3Kg\", \"Squat4Kg\"]`` columns using the DataFrame's built-in methods and print them.**\n", - "- **Assign the maximum squat, bench, and deadlifts of every person to a new column in the DataFrame using the DataFrame's built-in methods.**\n", - "- **Negative values are failed lifts: replace the negative values in the maximum squat, bench and deadlift column with np.NaNs (import numpy if you need to).**\n", - "- **Drop the rows containing a NaN in any of the three max lifts with ``.dropna()`` (use the subset keyword argument).**\n", - "- **Finally, calculate the total weight lifted with ``.sum()`` and assign it to a new column**\n", - "\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": { - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Squat1Kg 157.123823\n", - "Squat2Kg 126.244905\n", - "Squat3Kg 25.316470\n", - "Squat4Kg NaN\n", - "dtype: float64 \n", - "\n", - "Squat1Kg -442.5\n", - "Squat2Kg -440.0\n", - "Squat3Kg -478.0\n", - "Squat4Kg NaN\n", - "dtype: float64 \n", - "\n", - "Squat1Kg 450.0\n", - "Squat2Kg 470.0\n", - "Squat3Kg 460.0\n", - "Squat4Kg NaN\n", - "dtype: float64 \n", - "\n", - " MaxSquat MaxBench MaxDeadlift Totalkg\n", - "0 225.5 165.0 270.5 661.0\n", - "1 215.0 152.5 205.0 572.5\n", - "2 200.0 140.0 220.0 560.0\n", - "3 185.0 140.0 215.0 540.0\n", - "4 210.0 110.0 215.0 535.0\n" - ] - } - ], - "source": [ - "print(df_ipf[[\"Squat1Kg\", \"Squat2Kg\", \"Squat3Kg\", \"Squat4Kg\"]].mean(axis=0), \"\\n\")\n", - "print(df_ipf[[\"Squat1Kg\", \"Squat2Kg\", \"Squat3Kg\", \"Squat4Kg\"]].min(axis=0), \"\\n\")\n", - "print(df_ipf[[\"Squat1Kg\", \"Squat2Kg\", \"Squat3Kg\", \"Squat4Kg\"]].max(axis=0), \"\\n\")\n", - "\n", - "df_ipf[\"MaxSquat\"] = df_ipf[[\"Squat1Kg\", \"Squat2Kg\", \"Squat3Kg\", \"Squat4Kg\"]].max(axis=1)\n", - "df_ipf[\"MaxBench\"] = df_ipf[[\"Bench1Kg\", \"Bench2Kg\", \"Bench3Kg\", \"Bench4Kg\"]].max(axis=1)\n", - "df_ipf[\"MaxDeadlift\"] = df_ipf[[\"Deadlift1Kg\", \"Deadlift2Kg\", \"Deadlift3Kg\", \"Deadlift4Kg\"]].max(axis=1)\n", - "\n", - "df_ipf.loc[df_ipf[\"MaxSquat\"] <= 0, \"MaxSquat\"] = np.nan\n", - "df_ipf.loc[df_ipf[\"MaxBench\"] <= 0, \"MaxBench\"] = np.nan\n", - "df_ipf.loc[df_ipf[\"MaxDeadlift\"] <= 0, \"MaxDeadlift\"] = np.nan\n", - "\n", - "df_ipf.dropna(subset=[\"MaxSquat\", \"MaxBench\", \"MaxDeadlift\"], inplace=True)\n", - "\n", - "df_ipf['Totalkg'] = df_ipf[[\"MaxSquat\", \"MaxBench\", \"MaxDeadlift\"]].sum(axis=1)\n", - "\n", - "print(df_ipf[[\"MaxSquat\", \"MaxBench\", \"MaxDeadlift\", \"Totalkg\"]].head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's interesting to see that the highest lift in the squat on the second try was higher than the third try. You expect every try to be higher than the previous. This is probably because that person failed his third attempt (the -478).\n", - "\n", - "There is an easier way to get descriptives fast. You can also use the ``.describe()`` method to get a quick overview of your DataFrame.\n", - "\n", - "- **Use the .describe method, what do you notice?**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AgeBodyweightKgSquat1KgSquat2KgSquat3KgSquat4KgBench1KgBench2KgBench3KgBench4KgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgMaxSquatMaxBenchMaxDeadliftTotalkg
count1323.0000001323.0000001323.0000001316.0000001300.0000000.01323.0000001320.0000001307.0000000.01323.0000001313.0000001284.0000000.01323.0000001323.0000001323.0000001323.000000
mean30.70899583.001814164.027211133.78761432.162692NaN120.50302388.0920457.450650NaN205.473167137.878142-55.096573NaN212.914966137.458428234.457294584.830688
std6.10169528.533200134.962333177.735143225.649644NaN70.600959116.638052149.537946NaN105.328889200.502987244.867722NaN71.93829953.20033467.287548186.447492
min18.50000042.950000-442.500000-405.000000-478.000000NaN-270.000000-292.500000-292.500000NaN-355.000000-380.000000-400.000000NaN67.50000040.00000090.000000210.000000
25%26.50000062.155000130.000000117.500000-187.500000NaN80.00000070.000000-132.500000NaN160.000000137.500000-272.500000NaN150.00000087.500000175.000000412.500000
50%29.50000073.850000190.000000180.000000132.500000NaN127.500000110.00000060.000000NaN215.000000195.000000-170.000000NaN212.500000137.500000235.000000587.500000
75%34.500000100.850000243.750000250.000000232.500000NaN170.000000167.500000150.000000NaN275.000000282.500000190.000000NaN265.000000180.000000292.500000741.250000
max70.500000191.500000450.000000470.000000460.000000NaN277.500000282.000000290.000000NaN365.000000380.000000398.500000NaN470.000000290.000000398.5000001090.000000
\n", - "
" - ], - "text/plain": [ - " Age BodyweightKg Squat1Kg Squat2Kg Squat3Kg \\\n", - "count 1323.000000 1323.000000 1323.000000 1316.000000 1300.000000 \n", - "mean 30.708995 83.001814 164.027211 133.787614 32.162692 \n", - "std 6.101695 28.533200 134.962333 177.735143 225.649644 \n", - "min 18.500000 42.950000 -442.500000 -405.000000 -478.000000 \n", - "25% 26.500000 62.155000 130.000000 117.500000 -187.500000 \n", - "50% 29.500000 73.850000 190.000000 180.000000 132.500000 \n", - "75% 34.500000 100.850000 243.750000 250.000000 232.500000 \n", - "max 70.500000 191.500000 450.000000 470.000000 460.000000 \n", - "\n", - " Squat4Kg Bench1Kg Bench2Kg Bench3Kg Bench4Kg Deadlift1Kg \\\n", - "count 0.0 1323.000000 1320.000000 1307.000000 0.0 1323.000000 \n", - "mean NaN 120.503023 88.092045 7.450650 NaN 205.473167 \n", - "std NaN 70.600959 116.638052 149.537946 NaN 105.328889 \n", - "min NaN -270.000000 -292.500000 -292.500000 NaN -355.000000 \n", - "25% NaN 80.000000 70.000000 -132.500000 NaN 160.000000 \n", - "50% NaN 127.500000 110.000000 60.000000 NaN 215.000000 \n", - "75% NaN 170.000000 167.500000 150.000000 NaN 275.000000 \n", - "max NaN 277.500000 282.000000 290.000000 NaN 365.000000 \n", - "\n", - " Deadlift2Kg Deadlift3Kg Deadlift4Kg MaxSquat MaxBench \\\n", - "count 1313.000000 1284.000000 0.0 1323.000000 1323.000000 \n", - "mean 137.878142 -55.096573 NaN 212.914966 137.458428 \n", - "std 200.502987 244.867722 NaN 71.938299 53.200334 \n", - "min -380.000000 -400.000000 NaN 67.500000 40.000000 \n", - "25% 137.500000 -272.500000 NaN 150.000000 87.500000 \n", - "50% 195.000000 -170.000000 NaN 212.500000 137.500000 \n", - "75% 282.500000 190.000000 NaN 265.000000 180.000000 \n", - "max 380.000000 398.500000 NaN 470.000000 290.000000 \n", - "\n", - " MaxDeadlift Totalkg \n", - "count 1323.000000 1323.000000 \n", - "mean 234.457294 584.830688 \n", - "std 67.287548 186.447492 \n", - "min 90.000000 210.000000 \n", - "25% 175.000000 412.500000 \n", - "50% 235.000000 587.500000 \n", - "75% 292.500000 741.250000 \n", - "max 398.500000 1090.000000 " - ] - }, - "execution_count": 69, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_ipf.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "What you could or should have noticed is that we lost a bunch of columns. Describe, by default, only included the columns with numerical values. If you also want the descriptives for the other columns we need to motivate Pandas a bit. \n", - "\n", - "- **Look up the documentation for describe and make sure all columns are described.**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
NameSexEquipmentAgeBirthYearClassDivisionBodyweightKgWeightClassKgSquat1KgSquat2KgSquat3KgSquat4KgBench1KgBench2KgBench3KgBench4KgDeadlift1KgDeadlift2KgDeadlift3KgDeadlift4KgDateMaxSquatMaxBenchMaxDeadliftTotalkg
count1323132313231323.000000132313231323.00000013231323.0000001316.0000001300.0000000.01323.0000001320.0000001307.0000000.01323.0000001313.0000001284.0000000.013231323.0000001323.0000001323.0000001323.000000
unique80021NaN61NaN15NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN7NaNNaNNaNNaN
topDariusz WszołaMRawNaN24-39OpenNaN74NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN2019-06-04NaNNaNNaNNaN
freq77431323NaN11721323NaN116NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN249NaNNaNNaNNaN
meanNaNNaNNaN30.708995NaNNaN83.001814NaN164.027211133.78761432.162692NaN120.50302388.0920457.450650NaN205.473167137.878142-55.096573NaNNaN212.914966137.458428234.457294584.830688
stdNaNNaNNaN6.101695NaNNaN28.533200NaN134.962333177.735143225.649644NaN70.600959116.638052149.537946NaN105.328889200.502987244.867722NaNNaN71.93829953.20033467.287548186.447492
minNaNNaNNaN18.500000NaNNaN42.950000NaN-442.500000-405.000000-478.000000NaN-270.000000-292.500000-292.500000NaN-355.000000-380.000000-400.000000NaNNaN67.50000040.00000090.000000210.000000
25%NaNNaNNaN26.500000NaNNaN62.155000NaN130.000000117.500000-187.500000NaN80.00000070.000000-132.500000NaN160.000000137.500000-272.500000NaNNaN150.00000087.500000175.000000412.500000
50%NaNNaNNaN29.500000NaNNaN73.850000NaN190.000000180.000000132.500000NaN127.500000110.00000060.000000NaN215.000000195.000000-170.000000NaNNaN212.500000137.500000235.000000587.500000
75%NaNNaNNaN34.500000NaNNaN100.850000NaN243.750000250.000000232.500000NaN170.000000167.500000150.000000NaN275.000000282.500000190.000000NaNNaN265.000000180.000000292.500000741.250000
maxNaNNaNNaN70.500000NaNNaN191.500000NaN450.000000470.000000460.000000NaN277.500000282.000000290.000000NaN365.000000380.000000398.500000NaNNaN470.000000290.000000398.5000001090.000000
\n", - "
" - ], - "text/plain": [ - " Name Sex Equipment Age BirthYearClass Division \\\n", - "count 1323 1323 1323 1323.000000 1323 1323 \n", - "unique 800 2 1 NaN 6 1 \n", - "top Dariusz Wszoła M Raw NaN 24-39 Open \n", - "freq 7 743 1323 NaN 1172 1323 \n", - "mean NaN NaN NaN 30.708995 NaN NaN \n", - "std NaN NaN NaN 6.101695 NaN NaN \n", - "min NaN NaN NaN 18.500000 NaN NaN \n", - "25% NaN NaN NaN 26.500000 NaN NaN \n", - "50% NaN NaN NaN 29.500000 NaN NaN \n", - "75% NaN NaN NaN 34.500000 NaN NaN \n", - "max NaN NaN NaN 70.500000 NaN NaN \n", - "\n", - " BodyweightKg WeightClassKg Squat1Kg Squat2Kg Squat3Kg \\\n", - "count 1323.000000 1323 1323.000000 1316.000000 1300.000000 \n", - "unique NaN 15 NaN NaN NaN \n", - "top NaN 74 NaN NaN NaN \n", - "freq NaN 116 NaN NaN NaN \n", - "mean 83.001814 NaN 164.027211 133.787614 32.162692 \n", - "std 28.533200 NaN 134.962333 177.735143 225.649644 \n", - "min 42.950000 NaN -442.500000 -405.000000 -478.000000 \n", - "25% 62.155000 NaN 130.000000 117.500000 -187.500000 \n", - "50% 73.850000 NaN 190.000000 180.000000 132.500000 \n", - "75% 100.850000 NaN 243.750000 250.000000 232.500000 \n", - "max 191.500000 NaN 450.000000 470.000000 460.000000 \n", - "\n", - " Squat4Kg Bench1Kg Bench2Kg Bench3Kg Bench4Kg \\\n", - "count 0.0 1323.000000 1320.000000 1307.000000 0.0 \n", - "unique NaN NaN NaN NaN NaN \n", - "top NaN NaN NaN NaN NaN \n", - "freq NaN NaN NaN NaN NaN \n", - "mean NaN 120.503023 88.092045 7.450650 NaN \n", - "std NaN 70.600959 116.638052 149.537946 NaN \n", - "min NaN -270.000000 -292.500000 -292.500000 NaN \n", - "25% NaN 80.000000 70.000000 -132.500000 NaN \n", - "50% NaN 127.500000 110.000000 60.000000 NaN \n", - "75% NaN 170.000000 167.500000 150.000000 NaN \n", - "max NaN 277.500000 282.000000 290.000000 NaN \n", - "\n", - " Deadlift1Kg Deadlift2Kg Deadlift3Kg Deadlift4Kg Date \\\n", - "count 1323.000000 1313.000000 1284.000000 0.0 1323 \n", - "unique NaN NaN NaN NaN 7 \n", - "top NaN NaN NaN NaN 2019-06-04 \n", - "freq NaN NaN NaN NaN 249 \n", - "mean 205.473167 137.878142 -55.096573 NaN NaN \n", - "std 105.328889 200.502987 244.867722 NaN NaN \n", - "min -355.000000 -380.000000 -400.000000 NaN NaN \n", - "25% 160.000000 137.500000 -272.500000 NaN NaN \n", - "50% 215.000000 195.000000 -170.000000 NaN NaN \n", - "75% 275.000000 282.500000 190.000000 NaN NaN \n", - "max 365.000000 380.000000 398.500000 NaN NaN \n", - "\n", - " MaxSquat MaxBench MaxDeadlift Totalkg \n", - "count 1323.000000 1323.000000 1323.000000 1323.000000 \n", - "unique NaN NaN NaN NaN \n", - "top NaN NaN NaN NaN \n", - "freq NaN NaN NaN NaN \n", - "mean 212.914966 137.458428 234.457294 584.830688 \n", - "std 71.938299 53.200334 67.287548 186.447492 \n", - "min 67.500000 40.000000 90.000000 210.000000 \n", - "25% 150.000000 87.500000 175.000000 412.500000 \n", - "50% 212.500000 137.500000 235.000000 587.500000 \n", - "75% 265.000000 180.000000 292.500000 741.250000 \n", - "max 470.000000 290.000000 398.500000 1090.000000 " - ] - }, - "execution_count": 70, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_ipf.describe(include='all')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With this information we can gain a better understanding of our data. We see that the majority of the data is on Men and that our slicing on Division has worked as there is only one value for those present.\n", - "\n", - "## Categories\n", - "\n", - "Pandas understands the concept of categorical variables. One advantage of using categorical variables is the reduced load on memory. Instead of storing a lot of string objects, Pandas can just store some integers and a hash-table for their values. The memory usage of a Categorical is proportional to the number of categories plus the length of the data. In contrast, an object dtype is a constant times the length of the data. Do some searching on the web to find out how we can inspect memory usage and how we can change the type of a column.\n", - "\n", - "### Assignment 12\n", - "\n", - "- **Check the data type of the Sex column with the accessor ``.dtype`` and the memory usage of the Sex column with ``.memory_usage()``.**\n", - "\n", - "- **Convert the column to a category with ``.astype()`` and check the memory usage again.**\n", - "\n", - "- **You probably did this in two lines of code (first convert, then check memory). Do the same, but use method chaining.**\n", - "\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "str\n", - "12073 \n", - "\n", - "1342 \n", - "\n", - "1342 \n", - "\n" - ] - } - ], - "source": [ - "print(df_ipf[\"Sex\"].dtype)\n", - "print(df_ipf[\"Sex\"].memory_usage(index=False, deep=True), \"\\n\")\n", - "\n", - "df_ipf[\"Sex\"] = df_ipf[\"Sex\"].astype(\"category\")\n", - "print(df_ipf[\"Sex\"].memory_usage(index=False, deep=True), \"\\n\")\n", - "\n", - "print(df_ipf[\"Sex\"].astype(\"category\").memory_usage(index=False, deep=True), \"\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we have all the years and total lifted weight. Only a few more steps and we can see the development of strength over the years!\n", - "\n", - "## Dates\n", - "\n", - "Dates are hard. Having a computer handle dates is difficult to say the least. Just consider the differences between datetime notation (e.g., MM/DD/YYYY vs. DD/MM/YYYY) between countries and languages, it becomes very complicated, very quickly. Python has a very useful standard library package for handling dates: [datetime](https://docs.python.org/3/library/datetime.html). Pandas can also make use of datetime objects. It even has a special accessor (.dt, remember the .str accessor?) for datetime objects so you easily extract the year, month or day!\n", - "\n", - "### Assignment 13\n", - "\n", - "- **Convert the date column to the datetime datatype with ``pd.to_datetime()``.** \n", - "\n", - "- **Assign the year to a new column named \"Year\".**\n", - "\n", - "- **Now access the column and print out the unique years (access it with ``.dt.year``).**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": { - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2014 2016 2017 2013 2015 2018 2019]\n" - ] - } - ], - "source": [ - "df_ipf[\"Date\"] = pd.to_datetime(df_ipf[\"Date\"])\n", - "\n", - "df_ipf[\"Year\"] = df_ipf[\"Date\"].dt.year\n", - "\n", - "print(df_ipf[\"Year\"].unique())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Iterating (repetition)\n", - "\n", - "During the pre-assignment you already learned to iterate over three lists, remember? We will give you a small recap and get more familiar with for loops. The general recipe is:\n", - "\n", - "```python\n", - "for in :\n", - " \n", - "```\n", - "\n", - "You also already learned to iterate over multiple iterables at the same time with the built-in functions ``zip()`` and ``enumerate()``. If you forgot about it, you can have a look at the pre-assignment again or use you best friend Google. However, what will you do if you need to loop over two iterables and you want to consider each combination of the two iterables? Although for loops are not the fastest, we want to show you how to make a nested for loop:\n", - "\n", - "```python\n", - "for in :\n", - " for in :\n", - " \n", - "```\n", - "\n", - "It will take the first var of the first loop, and then all var's of the second loop. Then it goes back to the first for loop and takes the second var, then runs again all the var's of the second loop. And so on..\n", - "\n", - "### Assignment 14\n", - "\n", - "So let's try out some loops!\n", - "````{margin}\n", - "```{admonition} Tip\n", - ":class: tip\n", - "use f-strings so we know what year we are printing!\n", - "```\n", - "````\n", - "- **Define the unique years of the df_ipf and print out the mean Totalkg score and the corresponding year**\n", - "- **We hope you noticed that the years are not in ascending order, please rearrange them and print again**\n", - "- **Print out the mean and maximal Totalkg score per gender**\n", - "- **Let's combine them: print out the mean Totalkg score per year and per gender**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": { - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2014: 574.156462585034\n", - "2016: 586.9358288770053\n", - "2017: 595.9348837209302\n", - "2013: 575.0641891891892\n", - "2015: 593.5153846153846\n", - "2018: 577.1730769230769\n", - "2019: 584.5642570281125\n", - "\n", - "\n", - "2013: 575.0641891891892\n", - "2014: 574.156462585034\n", - "2015: 593.5153846153846\n", - "2016: 586.9358288770053\n", - "2017: 595.9348837209302\n", - "2018: 577.1730769230769\n", - "2019: 584.5642570281125\n", - "\n", - "\n", - "M mean: 724.1729475100942\n", - "M max: 1090.0\n", - "F mean: 406.32844827586206\n", - "F max: 671.5\n", - "\n", - "\n", - "M, 2013 mean: 711.3295454545455\n", - "M, 2014 mean: 703.1494252873563\n", - "M, 2015 mean: 733.2368421052631\n", - "M, 2016 mean: 726.347619047619\n", - "M, 2017 mean: 723.51171875\n", - "M, 2018 mean: 727.7150537634409\n", - "M, 2019 mean: 735.5234375\n", - "F, 2013 mean: 375.2083333333333\n", - "F, 2014 mean: 387.1166666666667\n", - "F, 2015 mean: 396.8703703703704\n", - "F, 2016 mean: 408.4207317073171\n", - "F, 2017 mean: 408.235632183908\n", - "F, 2018 mean: 419.86516853932585\n", - "F, 2019 mean: 424.8719008264463\n" - ] - } - ], - "source": [ - "years = df_ipf['Year'].unique()\n", - "for i in years:\n", - " print(f\"{i}: {df_ipf['Totalkg'][df_ipf['Year'] == i].mean()}\")\n", - " if i == years[-1]: # last entry\n", - " print('\\n') # new line for readability\n", - "\n", - "years.sort() # OR years = sorted(years)\n", - "for i in years:\n", - " print(f\"{i}: {df_ipf['Totalkg'][df_ipf['Year'] == i].mean()}\")\n", - " if i == years[-1]:\n", - " print('\\n')\n", - "\n", - "sex = df_ipf['Sex'].unique()\n", - "for i in sex:\n", - " print(f\"{i} mean: {df_ipf['Totalkg'][df_ipf['Sex'] == i].mean()}\")\n", - " print(f\"{i} max: {df_ipf['Totalkg'][df_ipf['Sex'] == i].max()}\")\n", - " if i == sex[-1]:\n", - " print('\\n')\n", - " \n", - "for n in sex:\n", - " for i in years:\n", - " print(f\"{n}, {i} mean: {(df_ipf['Totalkg'][(df_ipf['Sex'] == n) & (df_ipf['Year'] == i)]).mean()}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Grouping data\n", - "\n", - "In the previous assignment you already calculated the mean Totalkg score per year and per gender. However, for loops are generally not fast and want to avoid them as much as possible and pandas often had built-in functions for this kind of stuff! We can use the [grouping](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.groupby.html) function. Showing it with an example is probably the fastest way!\n", - "\n", - "```Python\n", - "df = pd.DataFrame({'Animal' : ['Falcon', 'Falcon',\n", - " 'Parrot', 'Parrot'],\n", - " 'Max Speed' : [380., 370., 24., 26.]})\n", - "# Animal Max Speed\n", - "#0 Falcon 380.0\n", - "#1 Falcon 370.0\n", - "#2 Parrot 24.0\n", - "#3 Parrot 26.0\n", - "\n", - "df.groupby(['Animal'], observed=False).max()\n", - "# Max Speed\n", - "#Animal\n", - "#Falcon 380.0\n", - "#Parrot 26.0\n", - "```\n", - "We now calculated the maximal value per Animal, but we can also calculate the mean ``.mean()``, standard deviation ``.std()``, the index of the maximal value ``.idxmax()`` and many more. Lastly, pandas makes it really easy to combine multiple calculations with ``.agg()``[docs](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.agg.html):\n", - "```python\n", - "df.groupby(['Animal'], observed=False).agg(['max', 'mean'])\n", - "# Max Speed Mean Max Speed \n", - "#Animal\n", - "#Falcon 380.0 375.0\n", - "#Parrot 26.0 25.0\n", - " \n", - "```\n", - "\n", - "So we will repeat the last exercise but with groupby!\n", - "\n", - "### Assignment 15\n", - "\n", - "First look up what the `observed` arugment does and why it is needed. Use the ``.groupby()`` method to examine your data:\n", - "\n", - "- **Group the data for year and calculate the mean Totalkg per year. Print the result.**\n", - "- **Make a groupby object for the ``[\"Sex\", \"Year\"]`` columns and calculate the mean and max Totalkg, print the result.**\n", - "\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": { - "scrolled": true, - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Totalkg\n", - "Year \n", - "2013 575.064189\n", - "2014 574.156463\n", - "2015 593.515385\n", - "2016 586.935829\n", - "2017 595.934884\n", - "2018 577.173077\n", - "2019 584.564257\n", - " Totalkg \n", - " mean max\n", - "Sex Year \n", - "F 2013 375.208333 533.5\n", - " 2014 387.116667 560.0\n", - " 2015 396.870370 615.5\n", - " 2016 408.420732 646.0\n", - " 2017 408.235632 615.0\n", - " 2018 419.865169 671.5\n", - " 2019 424.871901 640.0\n", - "M 2013 711.329545 920.0\n", - " 2014 703.149425 972.5\n", - " 2015 733.236842 1000.5\n", - " 2016 726.347619 1043.0\n", - " 2017 723.511719 1090.0\n", - " 2018 727.715054 1083.5\n", - " 2019 735.523438 972.5\n" - ] - } - ], - "source": [ - "print(df_ipf[[\"Totalkg\", \"Year\"]].groupby(\"Year\", observed=False).mean())\n", - "\n", - "print(df_ipf[[\"Totalkg\", \"Sex\", \"Year\"]].groupby([\"Sex\", \"Year\"], observed=False).agg(['mean', 'max']))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## timeit\n", - "\n", - "Besides the fact that our code is much more cleaner with ``.groupby()``, for loops are time consuming. We can actually time how long it takes to execute the code with the [%timeit or %%timeit](https://ipython.readthedocs.io/en/stable/interactive/magics.html#magic-timeit) command. \n", - "%timeit works for one line and %%timeit for one cell block (and needs to be on top of the cell block!).\n", - "Example:\n", - "```python\n", - "%timeit 5 + 5\n", - "10.8 ns +- 0.798 ns per loop (mean +- std. dev. of 7 runs, 100000000 loops each)\n", - "\n", - "%%timeit\n", - "x = 5 + 5\n", - "z = x + 10\n", - "53 ns +- 8.4 ns per loop (mean +- std. dev. of 7 runs, 10000000 loops each)\n", - "```\n", - "\n", - "Let's compare the time needed for the nested for loop and for the groupby function!\n", - "\n", - "### Assignment 16\n", - "\n", - "- **Calculate the mean Totalkg per year and per gender again but time it, which way is faster?**\n", - "\n", - "You should get something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": { - "tags": [ - "hide-input", - "no-execute" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "8.56 ms ± 470 μs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "%%timeit\n", - "for n in sex:\n", - " for i in years:\n", - " f\"{n}, {i} mean: {(df_ipf['Totalkg'][(df_ipf['Sex'] == n) & (df_ipf['Year'] == i)]).mean()}\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": { - "tags": [ - "hide-input", - "no-execute" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "5.56 ms ± 509 μs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "\n", - "%timeit df_ipf[[\"Totalkg\", \"Sex\", \"Year\"]].groupby([\"Sex\", \"Year\"], observed=False).mean() " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "So the groupby function is twice as fast! These few ms do not look like a lot of time gain, but wait till you are going to work on real big data sets! \n", - "\n", - "But we are there! You calculated the total weight lifted per year and per gender. If we make a nice boxplot, it looks like this (you don't have to do this, this one is on the house):" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": { - "tags": [ - "hide-input" - ] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAANzdJREFUeJzt3Qd4lGW6//E7hBQCoUvJ0lcQaXsAFUWXIs1diiwe2RVEKUtXUUBYBAVcysIqQZEiivSmK9bDKrBSDoKILEhZ/6ASgiwlKL0mkPlf97NnxiQkkIQp7zvP93Ndw2Teeae9TCa/uZ8W4fF4PAIAAGCxAqF+AgAAAKFGIAIAANYjEAEAAOsRiAAAgPUIRAAAwHoEIgAAYD0CEQAAsF5B649ALqWnp8vhw4clPj5eIiIiOGwAALiATrd49uxZSUhIkAIFcq4DEYhyScNQxYoV/fX/AwAAguiHH36QChUq5Hg9gSiXtDLkPaBFixb1z/8OAAAIqDNnzpiChvfveE4IRLnkbSbTMEQgAgDAXW7U3YVO1QAAwHoEIgAAYD0CEQAAsB59iPzs6tWrkpaWFvZvrKioKImMjAz10wAAwC8IRH6c5+Do0aNy6tQpsUXx4sWlXLlyzMsEAHA9ApGfeMNQmTJlJC4uLqxDgoa/CxcuSEpKirlcvnz5UD8lAABuCoHIT81k3jBUqlQpsUGhQoXMuYYifd00nwEA3IxO1X7g7TOklSGbeF+vDX2mAADhjUDkR+HcTJYd214vACB80WQGAPB7N4KdO3fKiRMnpGTJklKvXj2a1eF4BCIAgN9s2LBBZsyYYQaaeOlo1AEDBkiTJk040nAsmswAAH4LQ6NHj5Zq1arJ9OnTZeXKleZcL+t2vd52Wj3bvn27/OMf/zDnehnOQIXIoXT01vPPPy9///vf5dixY1KiRAn51a9+JWPGjJF77rkn1E8PADLRP+xaGdLPp3HjxkmBAv/5vl27dm1zedSoUTJz5ky59957rW0+o3rmbFSIHOqhhx6Sr7/+WubPny/79u2TDz/8UJo1a2ba5AHAabTPkDaTde3a1ReGvPSybj9y5IjZz0ZUz5yPQORAOqfRxo0bZdKkSdK8eXOpXLmy3HXXXTJixAhp27at2ef06dPSp08fMwdQ0aJF5f777zcBSh0/fty02U+YMMF3n1u2bJHo6GhZtWpVyF4XgPDl/bJWtWrVbK/3brfxS13W6plWzXTaEm/1TLdr9Yzms9AiEDlQkSJFzOn999+Xy5cvZztTtAYj/TambfTbtm2TBg0aSIsWLcyHzS233CJvvfWWaV776quv5Ny5c/Loo4+aTo2tW7cOyWsCEN50NJlKSkrK9nrvdu9+NqF65g4EIgcqWLCgzJs3zzSX6Xph2ub+3HPP+UrNa9eulV27dsk777wjd9xxh1SvXl1eeukls+/f/vY3s89vf/tb6d27tylT9+vXT2JjY+Uvf/lLiF8ZgHClQ+u1Mr148WJJT0/PdJ1e1u26zI/uZxuqZ+5AIHJwH6LDhw+bvkNt2rSRdevWmSqQBiWtCGnVR5cJ8VaT9KTfwL7//nvffWhIunLlirz99tvmw0hDEQAEgnaU1ir05s2bTQfqPXv2mDUP9Vwv6/b+/ftb2aGa6pk7RHi0/QU3dObMGSlWrJjpu6N9djK6dOmSCSPaRh7I0PHHP/5RVq9ebT50pk2bZkJSVlolKl26tPlZP4i0gqRLa7z33nvSvn17vz6fYL1uAO4eSaWVIQ1Dts5DpH2DtFqv0w9kHIHnrZ5pYNTP0kWLFlkZGEP59zsjht27SK1atUy/Iq0U6YeNNq1VqVIl231TU1PNL+Dvf/97qVmzpvTq1cs0s5UtWzbozxuAPTT0aDM/M1VfWz3TuZg0/Ohns36R1BCk1Xutno0dO5YwFGJUiBxYIfrpp5/k4Ycflp49e5r29vj4eNM5+sknnzSdqd98803zoXP27FkzEu22224zzWvawbpjx46mKvTss8+a/kQ68kyb03S0mt7Pxx9/LP5ChQgAco/qWWhQIXIxDTCNGjWSxMRE0ydIm7wqVqxoOklr52pdVFXDz8iRI01o8g6z15CkFSBtSps6darpfO0NbwsXLjThSod2aukaABBcVM+cjQqRi/oQOY2trxsAEH4VIkaZAQAA6xGIAACA9QhEAADAegQiAABgPQIRAACwHoEIAABYj0AEAACsRyACAADWYy2zADp27JiZCCpYdOIp1ioD/D8B6cGDB/N9+0qVKjFxKeACBKIAhqFHuz0maamXJViiomNk0cIFeQpF3bt3l/nz51+z/dtvv5Vbb73Vz88QcB8NQ3369Mn37WfPni01atTw63MC4H8EogDRypCGoYvVmkp6bDEJtAKXTovsX28eN69VogceeEDmzp2badstt9zi52cIuJNWeDTUZCc5OVnGjx9v1hWsXLlyjrcH4HwEogDTMJReuLQ4WUxMjFkcFsC1dJ2+G1V4NAzZWAWiORHhhEAEAMgXmhMRTghEkI8//liKFCniOxK/+c1v5J133uHIALgumhMRTghEkObNm8vMmTN9R6Jw4cIcFQA3RHMiwgmBCCYAMaIMAGAzJmYEAADWIxABAADr0WQWjPmBwuhxAAAIRwSiAC6joTNH62SJwaKPp4+bF/PmzQvY8wEAwC0IRAGis0XrMhqsZQYAgPMRiAIcilhsFQAA5wtpp+oNGzZI+/btJSEhQSIiIuT999/PdL3H45ExY8aY6wsVKiTNmjWTPXv2ZNrn8uXL8uSTT0rp0qXN8PEOHTrIoUOHMu1z8uRJ6datm2lO0pP+fOrUqaC8RgAA4HwhDUTnz5+XX/3qV/Laa69le/3kyZNlypQp5vqtW7ea9bZatWolZ8+e9e3z9NNPy3vvvSfLli2TjRs3yrlz56Rdu3Zy9epV3z5dunSRHTt2yCeffGJO+rOGIgAAgJA3mekSEXrKjlaHpk6dalaR7tSpk9k2f/580wS1ZMkS6du3r+mfM2fOHFm4cKG0bNnS7LNo0SKpWLGirFmzRtq0aSPffPONCUFffPGFNGrUyOzzxhtvyD333CN79+6V2267LdvH18qTnrzOnDkTgCMAOAOLdAKwnWP7ECUlJcnRo0eldevWmVZlb9q0qWzatMkEom3btklaWlqmfbR5rU6dOmYfDUSbN282zWTeMKTuvvtus033ySkQTZw4UcaOHRvgVwk4A4t0ArCdYwORhiGVtVOyXk5OTvbtEx0dLSVKlLhmH+/t9bxMmTLX3L9u8+6TnREjRsjgwYMzVYi08gSEIxbpBGA7xwYiL+1snbUpLeu2rLLuk93+N7ofrUbpCbABi3QCsJ1jl+7QDtQqaxUnJSXFVzXSfVJTU80osuvtc+zYsWvu//jx4wyJBwAAzq4QVa1a1YSZ1atXS/369c02DT/r16+XSZMmmcsNGzaUqKgos0/nzp3NtiNHjsju3bvNCDWlnae18/WXX34pd911l9m2ZcsWs61x48YBfQ0axJiYEQAA5wtpINIh8t99912mjtQ6JL5kyZKmT4MOqZ8wYYJUr17dnPTnuLg4M4xeacfoXr16yZAhQ6RUqVLmdkOHDpW6dev6Rp3dfvvt8sADD0jv3r3l9ddfN9v69Oljhubn1KHaX2HosW6PyuXUNAmWmOgoWbBwUZ4qX927dzej97ST+qxZszJdN2DAAJk5c6Y8/vjjLPEBAHnAyE33CWkg+uqrr6R58+a+y95OzN4/wMOGDZOLFy+aP8zaLKYjxVatWiXx8fG+2yQmJkrBggVNhUj3bdGihbltZGSkb5/FixfLU0895RuNppM35jT3kb9oZUjDUL9aZyWh8M9zIgXK4fORMutf8eZx8zo7tnYW13mc9FjqBJjeX+alS5eaYAoAyBtGbrpPSAORzjytnZtzop2edaZqPV2vM+i0adPMKSdaOdL5iUJBw1CV+MAHopvRoEED2b9/v6xYsUK6du1qtunPGpSqVasW6qcHAK7DyE33cWwfIgRXjx49ZO7cub5A9NZbb0nPnj1l3bp1/FcAQB4xctN9HDvKDMGlS5no0icHDhww8zx9/vnn8uijj/LfAACwAhUiGLo4btu2bU0Ha23G1J91WzihkyMAuPtz2NscqRU4fyMQwUebyJ544gnz8/Tp08PuyNDJEQDc/TmsZs+eLTVq1BB/IxDBR6cn0LmelK4DF27o5AgAzv0cVtplY/z48WZh98qVK0t2AjX6mUAEH52q4JtvvvH9HG7o5AgAzv8cVhqGAlEFuh4CURDmB3LT4xQtWtQv9wMAgJsQiAJEZ9HWmaN1ssRg0cfTx80LncTyet5///2bfFYAADgfgShAdLZoXUaDtcwAAHA+AlGAQ1Fel9EAAADBx8SMAADAegQiAABgPQKRH11vodpwZNvrBQCELwKRH0RFRZnzCxcuiE28r9f7+gEAcCs6VfuBTmJYvHhxSUlJMZfj4uIkIiJCwrkypGFIX6++7nCcxBEAYBcCkZ+UK1fOnHtDkQ00DHlfNwAAbkYg8hOtCJUvX17KlCkjaWlpEu60mYzKEAAgXBCI/ExDAkEBAAB3oVM1AACwHoEIAABYj0AEAACsRyACAADWo1N1mLl06ZIcPHgw37evVKmSxMbG+vU5AQDgdASiMKNhqE+fPvm+/ezZs6VGjRp+fU4AADgdgSjMaIVHQ012kpOTZfz48TJy5EipXLlyjrcHAMA2BKIwo81dN6rwaBiiCgQAwM/oVA0AAKxHIAIAANYjEAEAAOsRiAAAgPUIRAAAwHoEIgAAYD0CEQAAsB6BCAAAWI9ABAAArEcgAgAA1iMQAQAA6xGIAACA9QhEAADAegQiAABgPQIRAACwHoEIAABYj0AEAACsRyACAADWIxABAADrEYgAAID1CEQAAMB6BCIAAGA9AhEAALAegQgAAFiPQAQAAKxHIAIAANYjEAEAAOsRiAAAgPUIRAAAwHoEIgAAYD0CEQAAsB6BCAAAWI9ABAAArEcgAgAA1iMQAQAA6xGIAACA9QpafwQAANd17NgxOX36dJ6OUnJycqbzvChWrJiULVuW/xUEFYEIAPijf90w9Fi3R+Vyalq+3ifjx4/P821ioqNkwcJFjg9F+QmKirDoTAQiANbjj37O9A++hqF+tc5KQuGrAX+vHD4fKbP+FW8e18mB6GbfM+EeFt3I0YHoypUrMmbMGFm8eLEcPXpUypcvL927d5dRo0ZJgQL/6f7k8Xhk7NixMnv2bDl58qQ0atRIpk+fLrVr1/bdz+XLl2Xo0KGydOlSuXjxorRo0UJmzJghFSpUCOGrA+AU/NG/MQ1DVeIDH4jcItjvGTeFRbdydCCaNGmSzJo1S+bPn28CzldffSU9evQw7cuDBg0y+0yePFmmTJki8+bNkxo1asi4ceOkVatWsnfvXomPjzf7PP300/LRRx/JsmXLpFSpUjJkyBBp166dbNu2TSIjI0P8KgE4BX/0wXvGXo4ORJs3b5YHH3xQ2rZtay5XqVLFVHk0GHmrQ1OnTpWRI0dKp06dzDYNT5qclyxZIn379jVJes6cObJw4UJp2bKl2WfRokVSsWJFWbNmjbRp0yaErxAILjrHAoALA9F9991nKkT79u0z1Z+vv/5aNm7caEKQSkpKMk1prVu39t0mJiZGmjZtKps2bTKBSKtAaWlpmfZJSEiQOnXqmH1yCkTazKYnrzNnzgT0tQKBRj8ZAHBpIBo+fLip8NSsWdM0bV29etV0QnvkkUfM9RqGVNa2VL3s7cWv+0RHR0uJEiWu2cd7++xMnDjR9E0CwgX9ZADApYFo+fLlpnlLm7+0D9GOHTtMfyCt8Dz++OO+/SIiIjLdTpvSsm7L6kb7jBgxQgYPHpypQqTNbIDb0U8GAFwWiJ599ln505/+JH/4wx/M5bp165rKj1ZvNBCVK1fObPeOQPNKSUnxVY10n9TUVDMCLWOVSPdp3Lhxjo+tTW96AgAA4T9Hk6MD0YULF3zD67206Sw9Pd38XLVqVRN4Vq9eLfXr1zfbNPysX7/ejFBTDRs2lKioKLNP586dzbYjR47I7t27zQg1AADgP26do8nRgah9+/bmoFSqVMk0mW3fvt0Mse/Zs6e5Xpu8tAltwoQJUr16dXPSn+Pi4qRLly6+xNirVy8z1F6H3JcsWdLMSaTVJu+oMwAAYPccTY4ORNOmTZPnn39eBgwYYJq4tO+Qjhx74YUXfPsMGzbMTLao+3gnZly1apVvDiKVmJgoBQsWNBUi78SMOm8RcxABABAYbuuv6OhApKFGh9h7h9lnR6tEOpu1nnISGxtrwpWewkWw55NRLLgIAAhXjg5EcM58Moo1dAAA4YpA5EJubZ8FAMCpCEQu5rb22WBheQoAQF4RiBBWWJ4CAJAfBCKEFZanAADkB4EIYYnmRABAXmSeBhoAAMBCBCIAAGA9AhEAALAegQgAAFiPQAQAAKxHIAIAANYjEAEAAOsRiAAAgPUIRAAAwHrMVA0AAPzu8PlIVz0WgQgAAPjdrH/Fi5sQiAAAgN/1q3XWrCsZrArRzQYwAhFgmWCVsYNZLgfgPAmFr0qV+OAEIn8gEAGWcVsZGwCCgUAEWCZYZWx/lLABIFgIRIBl3FbGDiaaEwF7EYgA4P9Q0UK4Dy1HzghEAPB/aE5EXhGiwweBCAD+D82JCPeh5cgZgQgAgHwiRIcPApGL0XYNINw+b+gng1AhELkYpVMAfN4AIQpEr776arbbIyIiJDY2Vm699VZp0qSJREbSGz7QaLsGECx0OEe4y3MgSkxMlOPHj8uFCxekRIkS4vF45NSpUxIXFydFihSRlJQUqVatmqxdu1YqVqwYmGcNg7ZrAMHC5w3CXYG83mDChAly5513yrfffis//fSTnDhxQvbt2yeNGjWSV155RQ4ePCjlypWTZ555JjDPGAAAINQVolGjRsm7774rv/zlL33btJnspZdekoceekj2798vkydPNj8DoUIHUABAQAPRkSNH5MqVK9ds121Hjx41PyckJMjZs2fzeteA39DhHAAQ0EDUvHlz6du3r7z55ptSv359s2379u3Sv39/uf/++83lXbt2SdWqVfN614Df0AEUABDQQDRnzhzp1q2bNGzYUKKionzVoRYtWpjrlHaufvnll/N614Df0AEUABDQQKQdplevXi179+41Jx1lVrNmTbntttsyVZEAAADCNhDt3LlT6tWrZwJQxhCk3n//fenYsaM/nx8AAIDzht23adPGjCTLSkeede3a1V/PCwCAgLp69arpA/uPf/zDnOtl2CvPFSLtPK39hTZt2iTly5c325YvXy49e/aUefPmBeI5AgDgVxs2bJAZM2b4Rkd7u4QMGDDArLYA++S5QvTCCy9Ihw4dpGXLlmZSxiVLlkiPHj1kwYIF8vDDDwfmWQIA4McwNHr0aLOqwvTp02XlypXmXC/rdr0e9slzIFI6I3WDBg3k7rvvlt69e8vSpUuZiBEA4HjaLKaVoXvuuUfGjRsntWvXNktP6ble1u0zZ86k+cxCuWoy+/DDD6/Zpp2n169fL4888ohZ2NW7j1aPAABwIh0YpM1kzz//vBQokLkmoJe1L+zAgQPNft659uDsFQP89Vi5CkTXGzn21ltvmZPSYESnNACAU2lXD5XT5MHe7d79kHfFihWTmOiooK8YoI+pjx3QQJSenp7vBwAAwClKlixpzpOSkkwzWVa6PeN+yLuyZcvKgoWL5PTp03m+bXJysowfP15GjhwplStXztNtNQzpYwdtlBkAAG6l8+jpaLLFixebPkMZm830y79u1xHUuh/yT4PJzYQTDUM1atSQYMpVIHr11VdzfYdPPfXUzTwfAAACJjIy0gyt19Fko0aNMn2GtJlMK0MahjZv3ixjx441+8EuuQpEiYmJuboz7UNEIAIQCtp/UTvCat8Pbe7Qb/j8UUN2dJ4hDT062kw7UHtpZUi3Mw+RnXIViLxtqgDgREyyh7zS0HPvvfcSouFDHyLAMsEaChusx/FOsqfzx+hQ6ozNH7qdb/zIiVYQGVqPmwpEhw4dMvMOHTx4UFJTUzNdN2XKlPzcJYAwHAp7s8Ng8zrJnreDrHeSPe0jopPsaSUgN81n4RYWgfy6amETdJ4DkS6Cp5Mv6rewvXv3Sp06deTAgQPi8XjM7NUAwmsobCiHwQZrkr1wDItAfm2wdJ23PAeiESNGyJAhQ+TFF1+U+Ph4s8p9mTJlzAfPAw88EJhnCSDkQ2FDMQw2WJPshWNYRHC4bTbmG9lgcRN0ngPRN998Y9YuMzcuWFAuXrwoRYoUMQHpwQcflP79+wfieQJAQCfZC7ewiMBy62zMwWyCDvtAVLhwYbl8+bL5OSEhQb7//nvfB9GPP/7o/2cIADlgkr3goX9VeMzGfD07LV/nLc+BSFe4//zzz6VWrVrStm1b03y2a9cuWbFihbkOAIKFSfYCj/5V4TUb8/WcsHydtzwHIh1Fdu7cOfPzmDFjzM/Lly+XW2+9NdcTOMI/wq3tGsgPJtkLLPpX2aOk5eu85TkQVatWzfdzXFycaW9EcIVj2zVwM5hkL7DoX2WHepav85avQLR161YpVapUpu2nTp0yw+7379/vz+cHh3xjU4yKgZMxyR5w879DAyxe5y3PgUjnHNKe6FlpR+t///vf/npeuAG+sdnJxsnScHN4zyAvmli8zluuA5HOTO316aefZmo60V84nbCxSpUq/n+GAKyeLA35x3sG+dHE0nXech2IOnbs6FvR/vHHH890XVRUlAlDL7/8sv+fIZAP4TZE2ObJ0pA/vGdwMyItXOct14FIO1Qp/SDWPkSlS5cO5PMC8iUchwjbPlka8o73DBCEPkTeYXeAE4XjEGHbJ0tD3vGeAYK02v369evlpZdeMst4aBPa7bffLs8++6z8+te/zs/dAX4Vbh3ObZ8sDXnHewa5denSJTl48GC+D1ilSpUkNjbWzkC0aNEi6dGjh3Tq1Emeeuops8r9pk2bpEWLFjJv3jzp0qVLYJ4pYCnbJ0tD3vGeQW5pGOrTp0++D9js2bMd9yUyaIFImxUmT54szzzzjG/boEGDzAzWf/7zn/0eiHQo//Dhw+Xvf/+7WUhWD/ycOXOkYcOG5noNZNqhVP9TTp48KY0aNZLp06dn+sOhUwIMHTrULEqr96HhTftkVKhQwa/PFQgE2ydLQ97xnskdpiT4T4VH/37mtyuB3j5c5DkQ6cSL7du3v2Z7hw4d5LnnnhN/0oCjHUWbN29uAlGZMmXMYrLFixf37aPhTMOYVqc0LOkfjFatWsnevXslPv4/HWuffvpp+eijj2TZsmVmQkldf61du3aybds2OqHC8WyfLA15x3vmxpiS4D+0uetGFZ7KDuxK4IhAVLFiRTPnkK5dlpFu0+v8adKkSeY+586d69uWca4jrQ5NnTrVpFdtwlPz5883/UeWLFkiffv2NZ1rtaK0cOFCadmypa/ZT+93zZo10qZNG78+ZyAQbJ4sDfnDeyZnTEmAmwpEPXv2lFdeecVUV7Tv0I4dO6Rx48amU/XGjRtNhUav9yedDFIDy8MPP2w6cv/iF78w35R79+5trtdvyDr6pnXr1r7bxMTESNOmTU2/Jg1EWgVKS0vLtE9CQoLUqVPH7JNTINJmNj15nTlzxq+vDcgrWydLQ/7xnrkWUxIgJ5nH8F6HVl60/03//v1N09OuXbtMU5T2H9q9e7dZ8V4DiD9p85zOr1K9enUzO3a/fv1MGFuwYIG53jtjb9YRRXrZe52eR0dHS4kSJXLcJzsTJ040w6m9J39Xv4CbmSxN+8HpOWEIvGfyNyWBNj3nNI3FkSNHzH6wS64rRNo85fW73/3OnAJNO4zecccdMmHCBHNZ/wDs2bPHhKTHHnvMt59WqbI+16zbsrrRPiNGjJDBgwdnqhARigDA3ZiSADddIVI3Chn+pv0jatWqlWmbznnknTNBR96orJWelJQUX9VI90lNTTUdtHPaJzva9Fa0aNFMJwBA+ExJkB2msbBXngKR9jLXN9P1Tv6k/SV0tFhG+/bt8w3/05E2GnhWr17tu17Dj/Y30v5NSofn61prGffRcqg283n3AQDYNyWBd0kqL6axsFueRpnpaJZArtmUlc51pKFFm8w6d+4sX375pZkvwTtnglastB+TXq/9jPSkP8fFxfnmQ9Ln26tXL9MZXIfca2jTOYnq1q3rG3UGALADUxLAL4HoD3/4g5kLKFjuvPNOee+990x/nhdffNFUhHSYvXZ68xo2bJjp7K2jz7wTM65atco3B5FKTEyUggULmlDlnZhRR8XRIRUA7MOUBLipQBTs/kNeOoGinq73vMaMGWNO15t4atq0aeYEALBLdut1abOZ/t349ttvzXx12pqgrQw60ky7ZoTrel3w8ygzAADcgvW64NdAlLXzGQAAbsB6XQjI0h0AALgJ63XB78PuAQAAwhGBCAAAWI9ABAAArEcfIgAI06HlecXwctiMQATXfoDz4Q34b2i50lUAdIkmwEYEIoQMc4MAzhla7r0PwFYEIoQMc4MgP6gsZo+h5cDNIRAhZPgAR35QWQQQCAQiAK5CZRFAIBCIAAeiWShnVBYBBAKBCHAgmoUAILgIRIAD0SwEAMFFIAIciGYhAAgulu4AAADWo0IUYHSOBQDA+QhEAUbnWAAAnI9AFGB0jgUAwPkIRAFG51gAAJyPTtUAAMB6BCIAAGA9AhEAALAegQgAAFiPQAQAAKxHIAIAANYjEAEAAOsRiAAAgPWYmBHAddfcS05OznSe04zsOgkpANzM+p6h/LwhEAHI1Zp748ePz/G62bNnS40aNTiSAPyyvmcoPm8IRACuu+Zebm8PAIH+rPHeRyAQiMIMTR8I1Jp7ABDOnzUEojBD0wcAAHlHIAozNH0AANzo6tWrsnPnTjlx4oSULFlS6tWrJ5GRkUF7fAJRmHFyORIAgOxs2LBBZsyYIUePHvVtK1eunAwYMECaNGkiwcA8RAAAIKRhaPTo0VKtWjWZPn26rFy50pzrZd2u1wcDgQgAAISsmUwrQ/fcc4+MGzdOateuLXFxceZcL+v2mTNnmv0CjUAEAABCQvsMaTNZ165dpUCBzJFEL+v2I0eOmP0CjT5EAAD4GVOg5I52oFZVq1bN9nrvdu9+gUQgAgAgzKdAOXbsmJw+fTpPt0nOxTIaOSlWrJiULVv2hvvpaDKVlJRkmsmy0u0Z9wskAhEAAGE8BYqGoUe7PSZpqZfzdfvx1wluOYmKjpFFCxfcMBTp0HodTbZ48WLTZyhjs1l6errZXr58ebNfoBGIAAAI4ylQtDKkYehitaaSHlss4I9X4NJpkf3rzePeKBDpPEM6tF5Hk40aNcr0GdJmMq0MaRjavHmzjB07NijzERGIAACwgIah9MKlxWmaNGliQo+ONhs4cKBvu1aGdHuw5iEiEAEAgJDS0HPvvfcyUzUQDIz6APidgnNFRkZK/fr1Q/b4VIhgDaeN+gDczmm/U04dSQV3IBDBGk4a9QGEAyf9Tjl5JBXcgUAEazhp1IcX32j9d1wU3/bt/Z1y8kgquAOBCAgRvtEG5rgovu3by6kjqeB8BCIgRPhG64zj4rZv+8GuKir6ysAGBCIgxPhGy3FxclVR0VcGNiAQAYBLpmugegYEDoEIAFw0tFxRVQT8j0AEAC4ZWg4gcAhEAOCSoeUAAodA5CfMJwMAgHsRiPyA+WQAAHA3ApEfMJ/M9VE9AwA4HYHIjxj5cS2qZwAANyAQIaCongEA3IBAhKCgegYAcLICoX4CAAAAoUaFCAAACxS4eCqsHsffCEQAAFigUNKGUD8FRyMQAQBggYtVm0h6oeJBqRAVcmH4clUgmjhxojz33HMyaNAgmTp1qtnm8Xhk7NixZq2hkydPSqNGjWT69OlSu3Zt3+0uX74sQ4cOlaVLl8rFixelRYsWMmPGDKlQoUIIXw0AOL9Jwm3NHzQL5UzDUHrh0kH833AX1wSirVu3mtBTr169TNsnT54sU6ZMkXnz5pn1hsaNGyetWrWSvXv3Snx8vNnn6aeflo8++kiWLVsmpUqVkiFDhki7du1k27ZtEhkZGaJXBAD548Zv38HCsUFYB6Jz585J165d5Y033jCBx0urQ1opGjlypHTq1Mlsmz9/vpQtW1aWLFkiffv2NfPgzJkzRxYuXCgtW7Y0+yxatEgqVqwoa9askTZt2mT7mFpV0pPXmTNnAv46AfyMKkjomz68/w9uChk0CyGsA9HAgQOlbdu2JtBkDERJSUly9OhRad26tW9bTEyMNG3aVDZt2mQCkVaB0tLSMu2TkJAgderUMfvkFIi0eU6b4gCEhpv+CAcbTR8cG1gYiLSZ65///KdpMstKw5DSilBGejk5Odm3T3R0tJQoUeKafby3z86IESNk8ODBmSpEWlUCEBxUQQAEk6MD0Q8//GA6UK9atUpiY2Nz3C8iIiLTZW1Ky7otqxvto5UmPQEIDaogAILJ0TNVa3NXSkqKNGzYUAoWLGhO69evl1dffdX87K0MZa306G2815UrV05SU1PNCLSc9gEAAHZzdIVIh8fv2rUr07YePXpIzZo1Zfjw4VKtWjUTeFavXi3169c312v40dA0adIkc1nDVFRUlNmnc+fOZtuRI0dk9+7dZoQaEGoMEwaA0HN0INJh89r5OaPChQubofPe7TqkfsKECVK9enVz0p/j4uKkS5cu5vpixYpJr169zFB7vV3JkiXNnER169b1jToDQonOwwAQeo4ORLkxbNgwM9nigAEDfBMzap8j7xxEKjEx0TSxaYXIOzGjzlvEHETBQxUkZwwTBoDQc10gWrduXabL2jF6zJgx5pQT7ZA9bdo0c0JoUAXJGZ2HASD0XBeI4E5UQQAgtApcOh1Wj+NvBCIEBVUQAAgN7UsbFR0jsn990B4zKjrGPK6bEIgAAAhjOsXMooULzFJWeZGcnCzjx483y2NVrlw5T7fVMOS2qW0IRAAAhDkNJ/kNKJUrVzaLp4c7AhEAIGzQTwb5RSDyI4aWA0Bo0E8GN4tA5EcMLQfcOVLFbaNiODbXop8MbhaByI8YWg6485u+W0bFcGyuj34yuBkEIj9iaDkQum/6NoyKCUUVxC3HBrhZBCIAYfVNP9xHxVAFAQKjQIDuFwAAwDUIRAAAwHo0mQEhxrwpABB6BCIgRJg3BQCcg0AEhAjzpgCAcxCIgBBixBAAOAOBCEFBPxkAgJMRiBBQ9JMBALgBgQgBRT8ZAIAbEIgQcPSTAQA4HRMzAgAA6xGIAACA9QhEAADAegQiAABgPTpV+xFz7QAA4E4EIj9grh0AANyNQOQHzLUDAIC7EYj8hLl2AABwLzpVAwAA6xGIAACA9QhEAADAegQiAABgPQIRAACwHoEIAABYj0AEAACsRyACAADWIxABAADrEYgAAID1CEQAAMB6rGUGAAhrly5dkoMHD2Z7XXJycqbz7FSqVEliY2MD9vzgDAQiAEBY0zDUp0+f6+4zfvz4HK+bPXu21KhRIwDPDE5CIAKAMHCzVZBwroTo69JQczO3R/gjEAFAGLjZKkg4V0I05IXj64J/EYgAIAzcbBXEex+wC/2rfkYgAoAwQBUE+UH/qp8RiAAH4lsbgGCgf9XPCESAA/GtDUAwUFn8GYEIIUMVJGd8awOA4CIQIWSoguSMb205I0gDCAQCEUKGKgjygyANIBAIRAgZqiDID4I0gEAgEAFwFYI0gEBgtXsAAGA9AhEAALAegQgAAFiPPkQBxhBhAACcj0AUYAwRBgDA+QhEAcYQYQAAnI9AFGAMEQYAwPnoVA0AAKxHhQgAwtjVq1dl586dcuLECSlZsqTUq1dPIiMjQ/20AMchEAFAmNqwYYPMmDFDjh496ttWrlw5GTBggDRp0iSkzw1wGprMACBMw9Do0aOlWrVqMn36dFm5cqU518u6Xa8H8LMIj8fjyXAZOThz5owUK1ZMTp8+LUWLFuU4AXB0M1nXrl1N+Bk3bpwUKPDzd9/09HQZNWqUJCUlyaJFi2g+Q9g7k8u/31SIACDMaJ8hbSbTUJQxDCm9rNuPHDli9gPggkA0ceJEufPOOyU+Pl7KlCkjHTt2lL1792baRwtcY8aMkYSEBClUqJA0a9ZM9uzZk2mfy5cvy5NPPimlS5eWwoULS4cOHeTQoUNBfjUAEBzagVpVrVo12+u92737AXB4IFq/fr0MHDhQvvjiC1m9erVcuXJFWrduLefPn/ftM3nyZJkyZYq89tprsnXrVtNhsFWrVnL27FnfPk8//bS89957smzZMtm4caOcO3dO2rVrZ8rKABBudDSZ0max7Hi3e/cD4LI+RMePHzeVIg1KOkJCn7pWhjTwDB8+3FcNKlu2rEyaNEn69u1r2gxvueUWWbhwofz+9783+xw+fFgqVqxoOhm2adMmV49NHyIAbkEfIiDM+xDpi8n67UfbybVq5BUTEyNNmzaVTZs2mcvbtm2TtLS0TPtoiKpTp45vn+xosNKDmPEEAG6g8wzp0PrNmzebDtTajeDChQvmXC/r9v79+9OhGnDjPERaDRo8eLDcd999Jswo79waWhHKSC8nJyf79omOjpYSJUpcs0/GuTmy6780duzYALwSAAg8raLrZ5jOQ6RdD7zKly9vtjMPEeDSQPTEE0+YERHaByiriIiIa8JT1m1Z3WifESNGmADmpRUibWYDALfQ0HPvvfcyUzUQLoFIR4h9+OGHZiKxChUq+LZrB2qllR791uOVkpLiqxrpPqmpqXLy5MlMVSLdp3Hjxjk+pja96QkA3N58Vr9+/VA/DcDxHN2HSKs4WhlasWKFfPbZZ9cMIdXLGnh0BJqXhh/tdO0NOw0bNpSoqKhM++j8G7t3775uIAIAAPZwdIVI272XLFkiH3zwgZmLyNvnR3uL65xD2uSlI8wmTJgg1atXNyf9OS4uTrp06eLbt1evXjJkyBApVaqU6ZA9dOhQqVu3rrRs2TLErxAAADiBowPRzJkzzblOtpjR3LlzpXv37ubnYcOGycWLF82ICm0Wa9SokaxatcoEKK/ExEQpWLCgdO7c2ezbokULmTdvHiMsAACA++YhCiXmIQIAwH3Cch4iAACAQCAQAQAA6xGIAACA9QhEAADAegQiAABgPUcPu3cS72A8FnkFAMA9vH+3bzSonkCUS2fPnjXnrGcGAIA7/47r8PucMA9RLqWnp8vhw4fNhI83Wjg20LwLzf7www/XnVPBRhwbjg3vG36n+KwJrTMO+xullSENQwkJCVKgQM49hagQ5ZIexIwLyzqBvtGc8GZzIo4Nx4b3Db9TfNaEVlEH/Y26XmXIi07VAADAegQiAABgPQKRC8XExMjo0aPNOTg2vG/4neLzhs9hJ4lx6d8oOlUDAADrUSECAADWIxABAADrEYgAAID1CEQAAMB6BKIQmDhxotx5551m1usyZcpIx44dZe/evdfMrDlmzBgzs2ahQoWkWbNmsmfPnkz7zJ4922zXia909uxTp05d81gdOnSQSpUqSWxsrJQvX166detmZtx2qmAeG6/Lly/Lf/3Xf5n9duzYIU4VzGNTpUoVc13G05/+9CdxqmC/b/7nf/5HGjVqZO6ndOnS0qlTJ7H92Kxbt+6a94z3tHXrVrH9fbNv3z558MEHzftF97v33ntl7dq14lQTg3hs/vnPf0qrVq2kePHiUqpUKenTp4+cO3dOgo1AFALr16+XgQMHyhdffCGrV6+WK1euSOvWreX8+fO+fSZPnixTpkyR1157zXyYlCtXzrxhvGuqqQsXLsgDDzwgzz33XI6P1bx5c3n77bfNG/ndd9+V77//Xv77v/9bnCqYx8Zr2LBh5hfa6YJ9bF588UU5cuSI7zRq1ChxqmAeG/090i8WPXr0kK+//lo+//xz6dKli9h+bBo3bpzp/aKnP/7xjyZc33HHHWL7+6Zt27bm/j/77DPZtm2b+RLWrl07OXr0qNh8bA4fPiwtW7aUW2+9VbZs2SKffPKJCVXdu3eXoPMg5FJSUnQJXs/69evN5fT0dE+5cuU8f/nLX3z7XLp0yVOsWDHPrFmzrrn92rVrze1Pnjx5w8f64IMPPBEREZ7U1FSPGwT62KxcudJTs2ZNz549e8x+27dv97hFII9N5cqVPYmJiR63CtSxSUtL8/ziF7/wvPnmmx63CtbnjX7GlClTxvPiiy96bD82x48fN9s3bNjg23bmzBmzbc2aNR6bj83rr79u3idXr171bdPPYd3322+/9QQTFSIHOH36tDkvWbKkOU9KSjLfGjSNe+kEV02bNpVNmzbl+3FOnDghixcvNt/koqKixPZjc+zYMendu7csXLhQ4uLixG0C/b6ZNGmSKV/rN9nx48dLamqq2H5stLT/73//26xtWL9+fdMM/Zvf/OaaZgInC9bnzYcffig//vhjaL7pO+zY6O/R7bffLgsWLDAVFq22vP7661K2bFlp2LCh2HxsLl++LNHR0ZkWXdXmN7Vx40YJJgJRiGkb7ODBg+W+++6TOnXqmG3eEqr+smSkl/NTXh0+fLgULlzY/FIePHhQPvjgA7H92Oh96wd1v379HFvOD+X7ZtCgQbJs2TLTx+GJJ56QqVOnyoABA8T2Y7N//35zrv0mtAnx448/lhIlSpg/AvqFw+mC8XnjNWfOHGnTpo1Z9dwNAnlstO+MNjtt377d9MnRPp2JiYmmeUj7zdh8bO6//36z/1//+lfzpevkyZO+5jVtdg0mAlGI6R+bnTt3ytKlS7P9Jcr6psy6LTeeffZZ84u4atUqiYyMlMcee8zcl83HZtq0aXLmzBkZMWKEuFGg3zfPPPOM+SNfr1490w9k1qxZ5g/cTz/9JDYfm/T0dHM+cuRIeeihh8y3+7lz55r7eOedd8TpgvF5ow4dOiSffvqp9OrVS9wikMdG99cvFNo5+X//93/lyy+/NB2stQ9RsP/oO+3Y1K5dW+bPny8vv/yyqdRrP6Rq1aqZYKV/r4KJQBRCTz75pCkr67fwChUq+LbrG0JlTdkpKSnXpPHc0FENNWrUMJ3d9Fv/ypUrTUc5m4+NdmzUY6Al3oIFC5oOfUqrRY8//rg4WbDeNxndfffd5vy7774Tm4+NNpGpWrVq+bbpe0g/wLX66mTBfN9oSNSKtI5ydYNgfN5oNVE/f3V0WYMGDWTGjBmmaUjDgO3vmy5dupj70eZo/dKlFdjjx49L1apVJZgIRCGgCVoT94oVK8wvStb/dL2sbzYtsXppKVF7/Wv/n5t9bG+7rc3H5tVXXzUjhHSYvZ40JKrly5eb/jJOFMr3jVYYMwYCW4+NVoQ0AGUcfpyWliYHDhyQypUrixMF+32jj6eBSCvRTu+rGKxjoyOtVMZ+Mt7L3qqj03hC8HmjQapIkSLmc1ibFfVLfFAFtQs3jP79+5ue+OvWrfMcOXLEd7pw4YLvCGnPfd1nxYoVnl27dnkeeeQRT/ny5c3IBC+9jfbGf+ONN3wjGPTyTz/9ZK7fsmWLZ9q0aWbbgQMHPJ999pnnvvvu8/zyl780owFsPjZZJSUlOX6UWbCOzaZNmzxTpkwx2/bv3+9Zvny5JyEhwdOhQwePUwXzfTNo0CAz0uzTTz/1/L//9/88vXr1MqNkTpw44XGiYP9O6agpvf5f//qXx+mCdWx0lFmpUqU8nTp18uzYscOzd+9ez9ChQz1RUVHmsu3vm2nTpnm2bdtmjstrr73mKVSokOeVV14J+msmEIWAvimyO82dO9e3jw5pHD16tBnWGBMT42nSpIl5w2Wk11/vfnbu3Olp3ry5p2TJkuY+qlSp4unXr5/n0KFDHtuPjRsDUbCOjX4wNWrUyHzQxcbGem677TZzm/Pnz3ucKpjvGx1OPmTIEBOC4uPjPS1btvTs3r3b41TB/p3SP4qNGzf2uEEwj83WrVs9rVu3Np/H+r65++67zbQfTiVBPDbdunUzxyU6OtpTr149z4IFCzyhEKH/BLcmBQAA4Cz0IQIAANYjEAEAAOsRiAAAgPUIRAAAwHoEIgAAYD0CEQAAsB6BCAAAWI9ABAAArEcgAgAA1iMQAQgbOvF+y5YtpU2bNtdcp6uLFytWzPGr0gMIDQIRgLARERFhVlrfsmWLvP76677tSUlJMnz4cHnllVekUqVKfn1MXe0egPsRiACElYoVK5rgM3ToUBOEtGrUq1cvadGihdx1113y29/+VooUKSJly5aVbt26yY8//ui77SeffCL33XefFC9eXEqVKiXt2rWT77//3nf9gQMHTOh6++23pVmzZhIbGyuLFi0K0SsF4E8s7gogLHXs2FFOnTolDz30kPz5z3+WrVu3yh133CG9e/eWxx57TC5evGiqRleuXJHPPvvM3Obdd981gadu3bpy/vx5eeGFF0wI2rFjhxQoUMD8XLVqValSpYq8/PLLUr9+fYmJiZGEhIRQv1wAN4lABCAspaSkSJ06deSnn36Sv/3tb7J9+3bTlPbpp5/69jl06JCpKO3du1dq1KhxzX0cP35cypQpI7t27TL35Q1EU6dOlUGDBgX5FQEIJJrMAIQlDTJ9+vSR22+/XX73u9/Jtm3bZO3ataa5zHuqWbOm2dfbLKbnXbp0kWrVqknRokVN+FFZO2JrpQlAeCkY6icAAIFSsGBBc1Lp6enSvn17mTRp0jX7lS9f3pzr9VoxeuONN0wzmN5GK0OpqamZ9i9cuDD/aUCYIRABsEKDBg1MHyHt/+MNSRlp09o333xjRqf9+te/Nts2btwYgmcKIBRoMgNghYEDB8qJEyfkkUcekS+//FL2798vq1atkp49e8rVq1elRIkSZmTZ7Nmz5bvvvjMdrQcPHhzqpw0gSAhEAKygTWCff/65CT86caM2hWnHaJ2sUUeQ6WnZsmWmr5Fe98wzz8hf//rXUD9tAEHCKDMAAGA9KkQAAMB6BCIAAGA9AhEAALAegQgAAFiPQAQAAKxHIAIAANYjEAEAAOsRiAAAgPUIRAAAwHoEIgAAYD0CEQAAENv9fxd7Y4SezKLtAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import seaborn as sns\n", - "\n", - "sns.boxplot(x=\"Year\", y=\"Totalkg\", hue=\"Sex\", data=df_ipf)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There you have it! It seems that powerlifting got slightly more competitive in the female population over the years. Of course we cannot draw any statistical conclusions from this, but at least there is a tendency!\n", - "\n" - ] - } - ], - "metadata": { - "celltoolbar": "Tags", - "hide_input": false, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.12" - }, - "toc": { - "base_numbering": 1, - "nav_menu": { - "height": "234px", - "width": "160px" - }, - "number_sections": true, - "sideBar": true, - "skip_h1_title": true, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": { - "height": "calc(100% - 180px)", - "left": "10px", - "top": "150px", - "width": "288.717px" - }, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/practicals_jn_book/week_3/finalbook.ipynb b/practicals_jn_book/week_3/finalbook.ipynb index 8fb64a1..c71db3b 100644 --- a/practicals_jn_book/week_3/finalbook.ipynb +++ b/practicals_jn_book/week_3/finalbook.ipynb @@ -80,7 +80,7 @@ "source": [ "### Assignment 0: load and inspect data\n", "\n", - "The breast cancer dataset is available on the [GitHub](https://github.com/Alek050/big_data_practicals/tree/main/data/week_3). It's a binary DataFrame. Pandas also provides the convenient ``read_parquet()`` function. Parquet files are binary representations of tabular data, making it extremely fast with human readable formats such as csv and excel files. \n", + "The breast cancer dataset is available on the [GitHub](https://github.com/Alek050/big_data_practicals/tree/main/data/week_3). It's a binary DataFrame. Pandas also provides the convenient ``read_parquet()`` function. Parquet files are binary representations of tabular data, making it extremely fast compared with human readable formats such as csv and excel files. \n", "\n", "- **Read the data with the ``read_parquet()`` function and check what type it returns by using ``type()``.**\n", "- **Print the head of the DataFrame.**\n", @@ -534,7 +534,7 @@ "source": [ "Take a look at the distribution of mean_radius for the benign and malignant cases. To do so you have to plot a histogram for both in the same figure. Do this using the ``matplotlib.pyplot`` sublibrary (import on top). Remember to first define a figure and then plot a hist?\n", "\n", - "- **Create a histogram for mean_radius for the malignant and benign cases. Add a legend, xlabel and ylabel. Choose wheter you use interactive plotting or object-oriented plotting**\n", + "- **Create a histogram for mean_radius for the malignant and benign cases. Add a legend, xlabel and ylabel. Choose whether you use interactive plotting or object-oriented plotting**\n", "\n", "You can spice up your vanilla matplotlib with the built-in [stylesheets](https://matplotlib.org/gallery/style_sheets/style_sheets_reference.html).\n", "\n", diff --git a/practicals_jn_book/week_4/finalbook.ipynb b/practicals_jn_book/week_4/finalbook.ipynb index 3c6ce4a..93c7074 100644 --- a/practicals_jn_book/week_4/finalbook.ipynb +++ b/practicals_jn_book/week_4/finalbook.ipynb @@ -1673,7 +1673,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": { "tags": [ "hide-input" @@ -1690,6 +1690,10 @@ "y = df[y_col]\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n", + "scaler = StandardScaler()\n", + "scaler.fit(X_train)\n", + "X_train_scaled = scaler.transform(X_train)\n", + "X_test_scaled = scaler.transform(X_test)\n", "\n", "def train_evaluate_model(model, X_train, X_test, y_train, y_test):\n", " \"\"\" A function that calculates the R^2 and the RMSE based on a model and dataset\n", @@ -1715,8 +1719,8 @@ " \n", " return R2, RMSE\n", "\n", - "lasso_R2, lasso_RMSE = train_evaluate_model(lasso_model, X_train, X_test, y_train, y_test)\n", - "ridge_R2, ridge_RMSE = train_evaluate_model(ridge_model, X_train, X_test, y_train, y_test)" + "lasso_R2, lasso_RMSE = train_evaluate_model(lasso_model, X_train_scaled, X_test_scaled, y_train, y_test)\n", + "ridge_R2, ridge_RMSE = train_evaluate_model(ridge_model, X_train_scaled, X_test_scaled, y_train, y_test)" ] }, {