# Data Visualisation and Exploratory Analysis with ggplot2 ----
# Video 4 of R Programming: Zero to Expert
# Goal: turn tidy data into clear charts and clear findings.

# Quick recap and why we visualise ----
# Videos 1 to 3: base R, data structures, functions, dplyr, tidyr.
# Today: ggplot2, the plotting package inside the tidyverse.

library(tidyverse)

library(palmerpenguins)

# Why charts? Anscombe's quartet ----
# Four tiny datasets with almost identical summary statistics.

quartet <- anscombe |>
  pivot_longer(
    everything(),
    names_to = c(".value", "set"),
    names_pattern = "(.)(.)"
  )
quartet |> arrange(set) |> head(5)

quartet |>
  group_by(set) |>
  summarise(
    mean_x = mean(x),
    mean_y = round(mean(y), 2),
    sd_y   = round(sd(y), 2),
    r      = round(cor(x, y), 2)
  )

ggplot(quartet, aes(x = x, y = y)) +
  geom_point(size = 3, colour = "steelblue") +
  geom_smooth(method = "lm", formula = y ~ x, se = FALSE) +
  facet_wrap(~ set)

# The grammar of graphics ----
# Every ggplot = data + aesthetic mappings + geoms (layers).
# Layers are added with the plus sign.

ggplot(data = penguins)

ggplot(
  data = penguins,
  mapping = aes(x = flipper_length_mm, y = body_mass_g)
)

ggplot(
  data = penguins,
  mapping = aes(x = flipper_length_mm, y = body_mass_g)
) +
  geom_point()

peng <- penguins |> drop_na()
nrow(penguins)
nrow(peng)

p <- ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g))
class(p)

p + geom_point()

# Two classic mistakes ----

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g)) |>
  geom_point()

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g))
  + geom_point()

# Scatter plots: colour, shape, size and alpha ----
# Map a column inside aes(); set a fixed value outside aes().

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g,
                 colour = species)) +
  geom_point()

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g,
                 colour = species, shape = species)) +
  geom_point(size = 2)

ggplot(peng, aes(x = bill_length_mm, y = bill_depth_mm,
                 colour = species, size = body_mass_g)) +
  geom_point(alpha = 0.6)

ggplot(peng, aes(x = bill_length_mm, y = bill_depth_mm,
                 colour = "blue")) +
  geom_point()

ggplot(peng, aes(x = bill_length_mm, y = bill_depth_mm)) +
  geom_point(colour = "blue")

# Overplotting: too many points in one place

nrow(mpg)
nrow(distinct(mpg, displ, hwy))

ggplot(mpg, aes(x = displ, y = hwy)) +
  geom_point()

ggplot(mpg, aes(x = displ, y = hwy)) +
  geom_jitter(width = 0.1, height = 0.5, alpha = 0.6)

set.seed(42)
diamonds_small <- diamonds |> slice_sample(n = 2000)
nrow(diamonds)
nrow(diamonds_small)

ggplot(diamonds_small, aes(x = carat, y = price)) +
  geom_point(alpha = 0.2)

# Pause and try it
# Plot mpg: displ on x, hwy on y, coloured by class.

ggplot(mpg, aes(x = displ, y = hwy, colour = class)) +
  geom_point(size = 2)

# Trend lines with geom_smooth() ----

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g)) +
  geom_point(alpha = 0.5) +
  geom_smooth(method = "lm", formula = y ~ x)

ggplot(peng, aes(x = bill_length_mm, y = bill_depth_mm)) +
  geom_point(aes(colour = species)) +
  geom_smooth(method = "lm", formula = y ~ x,
              colour = "black")

ggplot(peng, aes(x = bill_length_mm, y = bill_depth_mm,
                 colour = species)) +
  geom_point() +
  geom_smooth(method = "lm", formula = y ~ x)

# Distributions: histograms and density plots ----
# One numeric variable: where are the values, how spread out?

ggplot(peng, aes(x = body_mass_g)) +
  geom_histogram()

ggplot(peng, aes(x = body_mass_g)) +
  geom_histogram(binwidth = 250, fill = "steelblue",
                 colour = "white")

ggplot(peng, aes(x = body_mass_g, fill = species)) +
  geom_histogram(binwidth = 250, alpha = 0.5,
                 position = "identity")

ggplot(peng, aes(x = body_mass_g, fill = species)) +
  geom_density(alpha = 0.4)

# A skewed distribution: diamond prices

ggplot(diamonds, aes(x = price)) +
  geom_histogram(binwidth = 500, fill = "grey40")

diamonds |>
  filter(price < 2500) |>
  ggplot(aes(x = price)) +
  geom_histogram(binwidth = 25)

diamonds |>
  mutate(band = floor(price / 50) * 50) |>
  count(band) |>
  filter(between(band, 1350, 1650))

# Pause and try it
# Histogram of hwy from mpg with binwidth = 2 and white outlines.

ggplot(mpg, aes(x = hwy)) +
  geom_histogram(binwidth = 2, colour = "white")

# Comparing groups: boxplots and violins ----

ggplot(peng, aes(x = species, y = body_mass_g)) +
  geom_boxplot()

ggplot(peng, aes(x = species, y = body_mass_g, fill = sex)) +
  geom_boxplot()

set.seed(1)
ggplot(peng, aes(x = species, y = flipper_length_mm)) +
  geom_violin(fill = "lightblue") +
  geom_jitter(width = 0.1, alpha = 0.4)

ggplot(mpg, aes(x = fct_reorder(class, hwy, median),
                y = hwy)) +
  geom_boxplot(fill = "wheat") +
  labs(x = "class")

# Bar charts: geom_bar() counts, geom_col() values ----

ggplot(peng, aes(x = species)) +
  geom_bar()

ggplot(peng, aes(x = species, fill = island)) +
  geom_bar()

ggplot(peng, aes(x = species, fill = island)) +
  geom_bar(position = "dodge")

ggplot(peng, aes(x = species, fill = island)) +
  geom_bar(position = position_dodge(preserve = "single"))

ggplot(peng, aes(x = island, fill = species)) +
  geom_bar(position = "fill")

mass_by_species <- peng |>
  group_by(species) |>
  summarise(mean_mass = mean(body_mass_g))
mass_by_species

ggplot(mass_by_species, aes(x = species, y = mean_mass)) +
  geom_bar()

ggplot(mass_by_species, aes(x = species, y = mean_mass)) +
  geom_col(fill = "darkseagreen")

ggplot(mpg, aes(x = fct_infreq(class))) +
  geom_bar(fill = "steelblue") +
  coord_flip() +
  labs(x = NULL, y = "number of cars")

class_counts <- mpg |> count(class, sort = TRUE)
ggplot(class_counts, aes(x = reorder(class, n), y = n)) +
  geom_col(fill = "grey50") +
  geom_text(aes(label = n), hjust = -0.3) +
  coord_flip() +
  labs(x = NULL, y = "cars")

# Pause and try it
# Show the share of each class within each drv (drive type).
# Hint: geom_bar() with position = "fill".

ggplot(mpg, aes(x = drv, fill = class)) +
  geom_bar(position = "fill")

# Line charts over time: the economics dataset ----

head(economics, 4)
range(economics$date)

ggplot(economics, aes(x = date, y = unemploy)) +
  geom_line()

economics |>
  mutate(rate = unemploy / pop * 100) |>
  ggplot(aes(x = date, y = rate)) +
  geom_line(colour = "firebrick", linewidth = 1)

econ_long <- economics |>
  select(date, psavert, uempmed) |>
  pivot_longer(-date, names_to = "measure",
               values_to = "value")
econ_long |> head(4)

ggplot(econ_long, aes(x = date, y = value,
                      colour = measure)) +
  geom_line()

# Facets: small multiples ----
# facet_wrap(~ var) for one variable, facet_grid(row ~ col) for two.

ggplot(econ_long, aes(x = date, y = value)) +
  geom_line() +
  facet_wrap(~ measure, ncol = 1, scales = "free_y")

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g)) +
  geom_point(colour = "steelblue", alpha = 0.7) +
  facet_wrap(~ species)

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g)) +
  geom_point(aes(colour = species), show.legend = FALSE) +
  facet_grid(sex ~ species)

ggplot(mpg, aes(x = hwy)) +
  geom_histogram(binwidth = 2) +
  facet_wrap(~ class, nrow = 2)

# Pause and try it
# Histogram of flipper_length_mm, one panel per island.

ggplot(peng, aes(x = flipper_length_mm, fill = species)) +
  geom_histogram(binwidth = 3) +
  facet_wrap(~ island, ncol = 1)

# Polishing: labels, themes and legends ----
# Exploration charts can be rough. Charts for others must be clear.

base_plot <- ggplot(peng, aes(x = flipper_length_mm,
                              y = body_mass_g,
                              colour = species)) +
  geom_point(size = 2, alpha = 0.8)
base_plot

labelled_plot <- base_plot +
  labs(
    title    = "Bigger flippers, heavier penguins",
    subtitle = "Palmer Archipelago, Antarctica, 2007-2009",
    x        = "Flipper length (mm)",
    y        = "Body mass (g)",
    colour   = "Species",
    caption  = "Source: palmerpenguins R package"
  )
labelled_plot

labelled_plot + theme_minimal()

labelled_plot + theme_bw(base_size = 14)

labelled_plot +
  theme_minimal(base_size = 13) +
  theme(
    legend.position  = "bottom",
    plot.title       = element_text(face = "bold"),
    panel.grid.minor = element_blank()
  )

labelled_plot +
  theme(legend.position = "bottom") +
  theme_minimal()

theme_set(theme_minimal(base_size = 13))

# Scales: colours, axis labels and log axes ----
# Every aesthetic has a scale: scale_<aesthetic>_<type>().

labelled_plot +
  scale_colour_brewer(palette = "Dark2")

ggplot(peng, aes(x = species, fill = island)) +
  geom_bar() +
  scale_fill_viridis_d()

ggplot(peng, aes(x = island, fill = species)) +
  geom_bar(position = "fill") +
  scale_y_continuous(labels = scales::percent) +
  labs(y = "share of penguins")

economics |>
  mutate(unemployed = unemploy * 1000) |>
  ggplot(aes(x = date, y = unemployed)) +
  geom_line() +
  scale_y_continuous(labels = scales::comma)

ggplot(diamonds_small, aes(x = carat, y = price)) +
  geom_point(alpha = 0.3) +
  scale_x_log10() +
  scale_y_log10(labels = scales::comma)

# Pause and try it
# Bar chart of mpg drv counts: viridis fill by class,
# a clear title, and the legend at the bottom.

ggplot(mpg, aes(x = drv, fill = class)) +
  geom_bar() +
  scale_fill_viridis_d() +
  labs(title = "Front-wheel drive is mostly compacts",
       x = "drive type", y = "cars", fill = NULL) +
  theme(legend.position = "bottom")

# Coordinates: zooming the right way ----
# Scale limits DROP data; coord_cartesian() just zooms.

ggplot(peng, aes(x = species, y = body_mass_g)) +
  geom_boxplot() +
  scale_y_continuous(limits = c(3000, 5000))

ggplot(peng, aes(x = species, y = body_mass_g)) +
  geom_boxplot() +
  coord_cartesian(ylim = c(3000, 5000))

# Annotations: guide the reader's eye ----

peak <- economics |> slice_max(unemploy, n = 1)
peak |> select(date, unemploy)

ggplot(economics, aes(x = date, y = unemploy / 1000)) +
  annotate("rect", xmin = as.Date("2007-12-01"),
           xmax = as.Date("2009-06-01"), ymin = -Inf,
           ymax = Inf, fill = "firebrick", alpha = 0.15) +
  geom_line() +
  geom_point(data = peak, colour = "firebrick", size = 3) +
  annotate("text", x = as.Date("2003-01-01"), y = 15.8,
           label = "Peak: Oct 2009", colour = "firebrick") +
  labs(x = NULL, y = "Unemployed (millions)",
       title = "US unemployment peaked after the 2008 crisis")

centres <- peng |>
  group_by(species) |>
  summarise(flipper_length_mm = mean(flipper_length_mm),
            body_mass_g       = mean(body_mass_g))
centres

ggplot(peng, aes(x = flipper_length_mm, y = body_mass_g,
                 colour = species)) +
  geom_point(alpha = 0.4) +
  geom_label(data = centres, aes(label = species),
             fontface = "bold", show.legend = FALSE) +
  theme(legend.position = "none")

# Saving plots with ggsave() ----

final_plot <- labelled_plot +
  geom_smooth(method = "lm", formula = y ~ x, se = FALSE) +
  scale_colour_brewer(palette = "Dark2") +
  theme(legend.position = "bottom")
final_plot

ggsave("penguins_flipper_mass.png", plot = final_plot,
       width = 8, height = 5, dpi = 300)
file.exists("penguins_flipper_mass.png")

# A full EDA workflow on penguins ----
# 1. Shape  2. Missing values  3. Distributions
# 4. Relationships  5. Groups  6. Outliers

dim(penguins)
glimpse(penguins)

colSums(is.na(penguins))
round(colMeans(is.na(penguins)) * 100, 1)

penguins |>
  filter(is.na(sex)) |>
  select(species, island, bill_length_mm, body_mass_g)

peng |>
  select(species, bill_length_mm:body_mass_g) |>
  pivot_longer(-species, names_to = "measure") |>
  ggplot(aes(x = value, fill = species)) +
  geom_histogram(bins = 25) +
  facet_wrap(~ measure, scales = "free")

peng |>
  select(bill_len = bill_length_mm, bill_dep = bill_depth_mm,
         flipper = flipper_length_mm, mass = body_mass_g) |>
  cor() |>
  round(2)

peng |>
  group_by(species, sex) |>
  summarise(
    n          = n(),
    mass_g     = round(mean(body_mass_g)),
    flipper_mm = round(mean(flipper_length_mm)),
    .groups    = "drop"
  )

# Outliers with the IQR rule
# Outlier if below Q1 - 1.5 * IQR or above Q3 + 1.5 * IQR.

q   <- quantile(peng$body_mass_g, c(0.25, 0.75))
iqr <- IQR(peng$body_mass_g)
c(lower = unname(q[1] - 1.5 * iqr),
  upper = unname(q[2] + 1.5 * iqr))
range(peng$body_mass_g)

mass_outliers <- peng |>
  group_by(species) |>
  mutate(
    q1 = quantile(body_mass_g, 0.25),
    q3 = quantile(body_mass_g, 0.75),
    is_outlier = body_mass_g < q1 - 1.5 * (q3 - q1) |
                 body_mass_g > q3 + 1.5 * (q3 - q1)
  ) |>
  ungroup() |>
  filter(is_outlier)
mass_outliers |> select(species, island, sex, body_mass_g)

set.seed(7)
ggplot(peng, aes(x = species, y = body_mass_g)) +
  geom_jitter(width = 0.15, alpha = 0.3) +
  geom_point(data = mass_outliers, colour = "red",
             size = 4, shape = 21, stroke = 1.5) +
  labs(title = "Two Chinstrap penguins are unusual",
       x = NULL, y = "Body mass (g)")

# Pause and try it
# Find flipper_length_mm outliers within each species.

peng |>
  group_by(species) |>
  filter(
    flipper_length_mm < quantile(flipper_length_mm, 0.25) -
      1.5 * IQR(flipper_length_mm) |
    flipper_length_mm > quantile(flipper_length_mm, 0.75) +
      1.5 * IQR(flipper_length_mm)
  ) |>
  select(species, sex, flipper_length_mm)

# Quick EDA on mpg: who breaks the pattern? ----

mpg |>
  filter(displ > 5, hwy > 22) |>
  select(manufacturer, model, year, displ, hwy, class)

two_seaters <- mpg |> filter(class == "2seater")
ggplot(mpg, aes(x = displ, y = hwy)) +
  geom_point(colour = "grey60") +
  geom_point(data = two_seaters, colour = "firebrick",
             size = 3) +
  annotate("text", x = 6.2, y = 30, label = "Corvettes",
           colour = "firebrick", fontface = "bold") +
  labs(x = "Engine size (litres)", y = "Highway mpg",
       title = "Sports cars beat the big-engine trend")

# Quick EDA on diamonds: a price paradox ----

ggplot(diamonds, aes(x = cut, y = price)) +
  geom_boxplot(fill = "lightblue") +
  scale_y_log10(labels = scales::dollar)

diamonds |>
  group_by(cut) |>
  summarise(n = n(),
            median_price = median(price),
            median_carat = median(carat))

diamonds |>
  filter(between(carat, 0.9, 1.1)) |>
  group_by(cut) |>
  summarise(n = n(), median_price = median(price))

diamonds |> filter(x == 0 | y == 0 | z == 0) |> nrow()
diamonds |>
  filter(y > 20 | z > 20) |>
  select(carat, price, x, y, z)

# Storytelling: from EDA to three clear findings ----

# Finding 1: Gentoo are the heaviest, about 5.1 kg vs 3.7 kg.
# Finding 2: Flipper length tracks body mass closely (r = 0.87).
# Finding 3: Bill length vs depth reverses when split by species.
# Each finding = one sentence + one chart + one number.

story_plot <- ggplot(peng, aes(x = bill_length_mm,
                               y = bill_depth_mm)) +
  geom_smooth(method = "lm", formula = y ~ x,
              colour = "grey40", linetype = "dashed",
              se = FALSE) +
  geom_point(aes(colour = species), alpha = 0.6) +
  geom_smooth(aes(colour = species), method = "lm",
              formula = y ~ x, se = FALSE) +
  scale_colour_brewer(palette = "Dark2") +
  labs(title = "The overall trend hides the real pattern",
       subtitle = "Dashed: all penguins. Coloured: each species",
       x = "Bill length (mm)", y = "Bill depth (mm)",
       colour = NULL) +
  theme(legend.position = "top")
story_plot

ggsave("finding3_simpsons_paradox.png", plot = story_plot,
       width = 8, height = 5, dpi = 300)
list.files(pattern = "png$")

# Recap ----
# Grammar: data + aes() + geoms, layered with +.
# Geoms: point, smooth, histogram, density, boxplot, bar, col, line.
# Polish: facets, labs(), themes, scales, coords, annotate, ggsave().
# EDA: shape, missing, distributions, relationships, groups, outliers.