# Statistics and Modelling in R ----
# Video 5: from describing data to testing ideas and building models.
# Datasets: penguins, mtcars, PlantGrowth, ToothGrowth and sleep.

# Setup and a quick recap ----
# Load the packages we need for the whole video.

library(tidyverse)

library(broom)

library(palmerpenguins)

peng <- penguins |>
  drop_na(body_mass_g, flipper_length_mm, sex)
nrow(penguins)
nrow(peng)

# Descriptive statistics ----
# Centre: mean and median. Spread: sd, var, quantiles and IQR.

mass <- peng$body_mass_g

mean(mass)
median(mass)
sd(mass)
var(mass)

quantile(mass)
quantile(mass, probs = c(0.1, 0.9))
IQR(mass)
range(mass)

peng |>
  group_by(species) |>
  summarise(
    n      = n(),
    mean   = mean(body_mass_g),
    median = median(body_mass_g),
    sd     = sd(body_mass_g),
    iqr    = IQR(body_mass_g)
  )

x <- c(12, 15, 14, 13, 16, 15, 14, 250)
mean(x)
median(x)

# Pause and try: mean, median and sd of flipper length
# for each sex of penguin.

peng |>
  group_by(sex) |>
  summarise(
    mean_flip   = mean(flipper_length_mm),
    median_flip = median(flipper_length_mm),
    sd_flip     = sd(flipper_length_mm)
  )

# Probability distributions ----
# d = density, p = probability below, q = quantile, r = random.

set.seed(42)
heights <- rnorm(1000, mean = 170, sd = 10)

head(round(heights, 1))
mean(heights)
sd(heights)

ggplot(tibble(height = heights), aes(x = height)) +
  geom_histogram(aes(y = after_stat(density)),
                 bins = 30, fill = 'grey80') +
  stat_function(fun = dnorm,
                args = list(mean = 170, sd = 10),
                colour = 'steelblue', linewidth = 1.2) +
  labs(title = '1000 simulated heights',
       x = 'Height (cm)', y = 'Density') +
  theme_minimal(base_size = 14)

pnorm(180, mean = 170, sd = 10)
1 - pnorm(190, mean = 170, sd = 10)
pnorm(180, 170, 10) - pnorm(160, 170, 10)

qnorm(0.5, mean = 170, sd = 10)
qnorm(0.95, mean = 170, sd = 10)
qnorm(c(0.025, 0.975))

dbinom(5, size = 10, prob = 0.5)
dbinom(0:10, size = 10, prob = 0.5) |> round(3)
pbinom(2, size = 10, prob = 0.5)

set.seed(1)
rbinom(8, size = 10, prob = 0.5)
runif(5, min = 0, max = 1) |> round(3)
sample(1:6, size = 10, replace = TRUE)

# Pause and try: if IQ is normal with mean 100 and sd 15,
# what share of people score above 130? What score is the top 1%?

1 - pnorm(130, mean = 100, sd = 15)
qnorm(0.99, mean = 100, sd = 15)

# Simulation: let the computer flip the coins ----
# Simulate coins, dice and the central limit theorem.

set.seed(2024)
flips <- sample(c('H', 'T'), size = 100, replace = TRUE)
table(flips)
mean(flips == 'H')

set.seed(2024)
n_flips <- 1:5000
heads <- rbinom(5000, size = 1, prob = 0.5)
running <- tibble(n = n_flips,
                  prop = cumsum(heads) / n_flips)

ggplot(running, aes(n, prop)) +
  geom_line(colour = 'steelblue') +
  geom_hline(yintercept = 0.5, linetype = 'dashed') +
  scale_x_log10() +
  labs(title = 'The law of large numbers',
       x = 'Number of flips (log scale)',
       y = 'Running proportion of heads') +
  theme_minimal(base_size = 14)

set.seed(7)
rolls <- sample(1:6, size = 6000, replace = TRUE)
table(rolls)
mean(rolls)

two_dice <- sample(1:6, 6000, TRUE) + sample(1:6, 6000, TRUE)
round(prop.table(table(two_dice)), 3)

# The central limit theorem
# Means of samples are roughly normal, even if the data is not.

set.seed(10)
waits <- rexp(5000, rate = 1 / 10)

ggplot(tibble(wait = waits), aes(wait)) +
  geom_histogram(bins = 40, fill = 'tomato') +
  labs(title = 'Raw data: very skewed waiting times',
       x = 'Minutes', y = 'Count') +
  theme_minimal(base_size = 14)

set.seed(10)
sample_means <- replicate(
  5000,
  mean(rexp(30, rate = 1 / 10))
)

mean(sample_means)
sd(sample_means)
10 / sqrt(30)

ggplot(tibble(m = sample_means), aes(m)) +
  geom_histogram(aes(y = after_stat(density)),
                 bins = 40, fill = 'grey80') +
  stat_function(fun = dnorm,
                args = list(mean = 10, sd = 10 / sqrt(30)),
                colour = 'steelblue', linewidth = 1.2) +
  labs(title = 'Means of 5000 samples of size 30',
       x = 'Sample mean (minutes)', y = 'Density') +
  theme_minimal(base_size = 14)

# Pause and try: repeat the experiment with samples of size 5.
# Is the standard error bigger or smaller? Is it still normal?

set.seed(10)
small_means <- replicate(5000, mean(rexp(5, rate = 1 / 10)))
sd(small_means)
10 / sqrt(5)
c(above_15 = mean(small_means > 15),
  below_5  = mean(small_means < 5))

# Sampling and confidence intervals ----
# Estimate a population value from a sample, with a margin of error.

adelie <- peng |> filter(species == 'Adelie')
flip <- adelie$flipper_length_mm

n    <- length(flip)
xbar <- mean(flip)
se   <- sd(flip) / sqrt(n)
c(n = n, mean = xbar, se = se)

t_crit <- qt(0.975, df = n - 1)
t_crit

lower <- xbar - t_crit * se
upper <- xbar + t_crit * se
round(c(lower = lower, upper = upper), 2)

tt <- t.test(flip)
tt$conf.int
tt$estimate

# What does 95% confidence really mean?
# Simulate 100 samples from a known population and count hits.

set.seed(99)
ci_sims <- map(1:100, function(i) {
  s <- rnorm(25, mean = 50, sd = 8)
  ci <- t.test(s)$conf.int
  tibble(sim = i, lower = ci[1], upper = ci[2])
}) |>
  list_rbind() |>
  mutate(hit = lower <= 50 & upper >= 50)

mean(ci_sims$hit)

ggplot(ci_sims, aes(y = sim, xmin = lower, xmax = upper,
                    colour = hit)) +
  geom_linerange(linewidth = 0.8) +
  geom_vline(xintercept = 50, linetype = 'dashed') +
  scale_colour_manual(values = c(`TRUE` = 'grey60',
                                 `FALSE` = 'red')) +
  labs(title = '100 confidence intervals, true mean = 50',
       x = 'Value', y = 'Simulation') +
  theme_minimal(base_size = 14)

# Hypothesis tests and p-values ----
# H0 = no effect. p-value: how surprising is our data if H0 is true?

set.seed(3)
fair_heads <- rbinom(10000, size = 100, prob = 0.5)

observed <- 61
mean(abs(fair_heads - 50) >= abs(observed - 50))
binom.test(observed, n = 100, p = 0.5)$p.value

# t-tests: comparing means ----
# One-sample, two-sample (Welch) and paired.

t.test(flip, mu = 192)

t.test(flipper_length_mm ~ sex, data = peng)

t.test(flipper_length_mm ~ species, data = peng)

peng |>
  filter(species %in% c('Adelie', 'Chinstrap')) |>
  t.test(flipper_length_mm ~ species, data = _)

# Paired t-test
# The sleep data: 10 patients, each tried two sleeping drugs.

sleep_wide <- sleep |>
  pivot_wider(names_from = group, values_from = extra,
              names_prefix = 'drug')
sleep_wide

t.test(sleep_wide$drug2, sleep_wide$drug1, paired = TRUE)

t.test(sleep_wide$drug2, sleep_wide$drug1)$p.value

# Checking assumptions
# t-tests assume roughly normal data, or a decent sample size.

ggplot(peng, aes(sample = flipper_length_mm)) +
  stat_qq(alpha = 0.6) +
  stat_qq_line(colour = 'red') +
  facet_wrap(~ sex) +
  labs(title = 'Normal QQ plots of flipper length') +
  theme_minimal(base_size = 14)

# Chi-square test of independence ----
# Are two categorical variables related?

tab <- table(peng$species, peng$island)
tab

chi <- chisq.test(tab)
chi
round(chi$expected, 1)

# Pause and try: is penguin sex related to species?
# Build the table and run the chi-square test.

sex_tab <- table(peng$species, peng$sex)
sex_tab
chisq.test(sex_tab)$p.value

# One-way ANOVA ----
# PlantGrowth: dried plant weight under a control and two treatments.

PlantGrowth |>
  group_by(group) |>
  summarise(n = n(), mean = mean(weight), sd = sd(weight))

ggplot(PlantGrowth, aes(group, weight, fill = group)) +
  geom_boxplot(alpha = 0.5, show.legend = FALSE) +
  geom_jitter(width = 0.1, size = 2) +
  labs(title = 'Plant weight by treatment',
       x = NULL, y = 'Dried weight') +
  theme_minimal(base_size = 14)

plant_aov <- aov(weight ~ group, data = PlantGrowth)
summary(plant_aov)

TukeyHSD(plant_aov)

# Pause and try: in ToothGrowth, does tooth length (len) differ
# between the three vitamin C doses? Use aov and TukeyHSD.

tooth <- ToothGrowth |> mutate(dose = factor(dose))
tooth_aov <- aov(len ~ dose, data = tooth)
summary(tooth_aov)
TukeyHSD(tooth_aov)$dose |> round(4)

# Correlation tests ----
# How strongly do two numeric variables move together?

cor(mtcars$wt, mtcars$mpg)
cor.test(mtcars$wt, mtcars$mpg)

cor.test(~ bill_length_mm + bill_depth_mm, data = peng)$estimate

peng |>
  group_by(species) |>
  summarise(r = cor(bill_length_mm, bill_depth_mm))

set.seed(123)
fake_p <- replicate(1000, t.test(rnorm(20), rnorm(20))$p.value)
mean(fake_p < 0.05)

# Simple linear regression ----
# Model mpg as a straight-line function of car weight.

ggplot(mtcars, aes(wt, mpg)) +
  geom_point(size = 2.5) +
  geom_smooth(method = 'lm', formula = y ~ x) +
  labs(title = 'Heavier cars use more fuel',
       x = 'Weight (1000 lbs)', y = 'Miles per gallon') +
  theme_minimal(base_size = 14)

fit1 <- lm(mpg ~ wt, data = mtcars)
fit1

summary(fit1)

# Tidy model output with broom
# tidy() = coefficients, glance() = one row of fit stats,
# augment() = data plus fitted values and residuals.

tidy(fit1, conf.int = TRUE)
glance(fit1) |> select(r.squared, adj.r.squared, sigma, AIC)

fit1_aug <- augment(fit1)
fit1_aug |>
  select(.rownames, mpg, wt, .fitted, .resid) |>
  arrange(desc(abs(.resid))) |>
  head(5)

# Regression diagnostics
# Check: linearity, constant spread, normal residuals, influence.

ggplot(fit1_aug, aes(.fitted, .resid)) +
  geom_point(size = 2.5) +
  geom_hline(yintercept = 0, linetype = 'dashed') +
  geom_smooth(method = 'loess', formula = y ~ x,
              se = FALSE, colour = 'red') +
  labs(title = 'Residuals vs fitted',
       x = 'Fitted mpg', y = 'Residual') +
  theme_minimal(base_size = 14)

fit1_aug |>
  select(.rownames, wt, mpg, .hat, .cooksd) |>
  slice_max(.cooksd, n = 3)

fit1_quad <- lm(mpg ~ wt + I(wt^2), data = mtcars)
tidy(fit1_quad) |> select(term, estimate, p.value)
glance(fit1_quad)$r.squared

# Predictions with predict()
# Pass new data with the same column names as the model.

new_cars <- tibble(wt = c(2, 3, 4.5))

predict(fit1, newdata = new_cars)
predict(fit1, newdata = new_cars, interval = 'confidence')
predict(fit1, newdata = new_cars, interval = 'prediction')

# Pause and try: fit mpg ~ hp. What is the slope, the R-squared,
# and the predicted mpg for a 150 horsepower car?

fit_hp <- lm(mpg ~ hp, data = mtcars)
tidy(fit_hp) |> select(term, estimate, p.value)
glance(fit_hp)$r.squared
predict(fit_hp, newdata = tibble(hp = 150))

# Multiple regression ----
# Many predictors: each slope 'holds the others fixed'.

fit2 <- lm(mpg ~ wt + hp, data = mtcars)
tidy(fit2, conf.int = TRUE) |>
  mutate(across(where(is.numeric), \(x) round(x, 3)))
glance(fit2) |> select(r.squared, adj.r.squared, sigma)

# Factors in a model
# A factor becomes dummy variables: each level vs a baseline.

tooth_fit <- lm(len ~ supp + dose, data = tooth)
tidy(tooth_fit) |> select(term, estimate, std.error, p.value)
levels(tooth$supp)

# Interactions
# Does the effect of one predictor depend on another? Use a * b.

tooth |>
  group_by(supp, dose) |>
  summarise(mean_len = mean(len), .groups = 'drop') |>
  ggplot(aes(dose, mean_len, colour = supp, group = supp)) +
  geom_point(size = 3) +
  geom_line(linewidth = 1) +
  labs(title = 'Orange juice helps most at low doses',
       x = 'Dose (mg/day)', y = 'Mean tooth length',
       colour = 'Supplement') +
  theme_minimal(base_size = 14)

tooth_int <- lm(len ~ supp * dose, data = tooth)
tidy(tooth_int) |>
  select(term, estimate, p.value) |>
  mutate(across(where(is.numeric), \(x) round(x, 4)))

# Comparing models ----
# Nested models: anova() F-test. Any models on the same data: AIC.

anova(tooth_fit, tooth_int)

AIC(tooth_fit, tooth_int)

fit3 <- lm(mpg ~ wt + hp + qsec + am, data = mtcars)
AIC(fit1, fit2, fit3)
anova(fit2, fit3)

# Logistic regression ----
# For yes/no outcomes: model the log-odds with glm(family = binomial).

peng_sex <- peng |>
  mutate(male = as.integer(sex == 'male')) |>
  select(male, sex, species, body_mass_g, bill_depth_mm)

ggplot(peng_sex, aes(body_mass_g, male, colour = species)) +
  geom_jitter(height = 0.05, alpha = 0.6) +
  geom_smooth(method = 'glm', formula = y ~ x, se = FALSE,
              method.args = list(family = binomial)) +
  labs(title = 'Probability a penguin is male',
       x = 'Body mass (g)', y = 'Male (1) or female (0)') +
  theme_minimal(base_size = 14)

logit1 <- glm(male ~ body_mass_g + bill_depth_mm + species,
              data = peng_sex, family = binomial)
tidy(logit1) |>
  mutate(across(where(is.numeric), \(x) signif(x, 3)))

exp(coef(logit1)) |> round(3)
exp(coef(logit1)['body_mass_g'] * 100)

new_peng <- tibble(
  species       = c('Adelie', 'Adelie', 'Gentoo'),
  body_mass_g   = c(3400, 4200, 5000),
  bill_depth_mm = c(17.5, 19, 15.5)
)
predict(logit1, newdata = new_peng, type = 'response') |>
  round(3)

peng_sex <- peng_sex |>
  mutate(prob = predict(logit1, type = 'response'),
         pred = if_else(prob > 0.5, 'male', 'female'))

conf_mat <- table(predicted = peng_sex$pred,
                  actual = peng_sex$sex)
conf_mat
mean(peng_sex$pred == peng_sex$sex)

# Train/test split
# Judge a model on data it has never seen.

set.seed(2025)
train_rows <- sample(nrow(peng_sex),
                     size = round(0.7 * nrow(peng_sex)))
train <- peng_sex[train_rows, ]
test  <- peng_sex[-train_rows, ]
c(train = nrow(train), test = nrow(test))

logit_train <- glm(male ~ body_mass_g + bill_depth_mm + species,
                   data = train, family = binomial)

test_prob <- predict(logit_train, newdata = test,
                     type = 'response')
test_pred <- if_else(test_prob > 0.5, 'male', 'female')
table(predicted = test_pred, actual = test$sex)
mean(test_pred == test$sex)

# Pause and try: in mtcars, model am (1 = manual) from wt.
# What is the odds ratio for wt? P(manual) for a 3000 lb car?

logit_am <- glm(am ~ wt, data = mtcars, family = binomial)
exp(coef(logit_am)) |> round(4)
predict(logit_am, newdata = tibble(wt = 3),
        type = 'response')

# Many models at once: nest + map + broom ----
# Fit the same model separately for each group.

by_species <- peng |>
  nest(data = -species)
by_species

species_models <- by_species |>
  mutate(
    model = map(data, \(d) lm(body_mass_g ~ flipper_length_mm,
                                data = d)),
    coefs = map(model, tidy),
    stats = map(model, glance)
  )

species_models |>
  unnest(coefs) |>
  filter(term == 'flipper_length_mm') |>
  select(species, estimate, std.error, p.value)

species_models |>
  unnest(stats) |>
  select(species, r.squared, sigma, nobs)

# Project: what predicts penguin body mass? ----
# Explore, build models step by step, compare, check, predict,
# and write a conclusion.

peng |>
  select(where(is.numeric), -year) |>
  cor() |>
  round(2)

m1 <- lm(body_mass_g ~ flipper_length_mm, data = peng)
m2 <- lm(body_mass_g ~ flipper_length_mm + species,
         data = peng)
m3 <- lm(body_mass_g ~ flipper_length_mm + species + sex,
         data = peng)
m4 <- lm(body_mass_g ~ flipper_length_mm + bill_length_mm +
           bill_depth_mm + species + sex, data = peng)

models <- list(m1 = m1, m2 = m2, m3 = m3, m4 = m4)

map(models, glance) |>
  list_rbind(names_to = 'model') |>
  select(model, r.squared, adj.r.squared, sigma, AIC) |>
  mutate(across(where(is.numeric), \(x) round(x, 3)))

anova(m3, m4)

tidy(m4, conf.int = TRUE) |>
  select(term, estimate, conf.low, conf.high, p.value) |>
  mutate(across(where(is.numeric), \(x) round(x, 3)))

augment(m4) |>
  ggplot(aes(.fitted, .resid, colour = species)) +
  geom_point(alpha = 0.7) +
  geom_hline(yintercept = 0, linetype = 'dashed') +
  labs(title = 'Model 4: residuals vs fitted',
       x = 'Fitted body mass (g)', y = 'Residual (g)') +
  theme_minimal(base_size = 14)

set.seed(42)
idx <- sample(nrow(peng), size = round(0.7 * nrow(peng)))
peng_train <- peng[idx, ]
peng_test  <- peng[-idx, ]

rmse <- function(model) {
  pred <- predict(model, newdata = peng_test)
  sqrt(mean((peng_test$body_mass_g - pred)^2))
}

formulas <- list(
  m1 = body_mass_g ~ flipper_length_mm,
  m3 = body_mass_g ~ flipper_length_mm + species + sex,
  m4 = body_mass_g ~ flipper_length_mm + bill_length_mm +
         bill_depth_mm + species + sex
)

map_dbl(formulas, \(f) rmse(lm(f, data = peng_train))) |>
  round(1)

new_bird <- tibble(flipper_length_mm = 215,
                   bill_length_mm = 47, bill_depth_mm = 15,
                   species = 'Gentoo', sex = 'male')
predict(m4, newdata = new_bird, interval = 'prediction') |>
  round(0)

# Conclusion
# 1. Flipper length is the strongest single predictor (r = 0.87).
# 2. Species and sex matter a lot: males are ~390 g heavier and
# Gentoo ~1000 g heavier than Adelie at the same size.
# 3. Model 4 explains ~87% of the variation; test error ~300 g.

# Recap ----
# Describe -> distributions -> simulate -> CIs -> tests -> models.
# lm for numbers, glm(binomial) for yes/no, broom to tidy,
# diagnostics to check, test data to judge predictions.