stanfit
object hold a lot of information and some of the elements that take of most of the memory are not necessarily needed in post-processing. This vignette introduces verbs that can safely remove these elements without corrupting the stanfit object.
rats <- rats_example(nCores = 1)
rats@stanmodel
The Model
S4 class stanmodel 'rats' coded as follows: data { int<lower=0> N; int<lower=0> T; real x[T]; real y[N,T]; real xbar; } parameters { real alpha[N]; real beta[N]; real mu_alpha; real mu_beta; // beta.c in original bugs model real<lower=0> sigmasq_y; real<lower=0> sigmasq_alpha; real<lower=0> sigmasq_beta; } transformed parameters { real<lower=0> sigma_y; // sigma in original bugs model real<lower=0> sigma_alpha; real<lower=0> sigma_beta; sigma_y = sqrt(sigmasq_y); sigma_alpha = sqrt(sigmasq_alpha); sigma_beta = sqrt(sigmasq_beta); } model { mu_alpha ~ normal(0, 100); mu_beta ~ normal(0, 100); sigmasq_y ~ inv_gamma(0.001, 0.001); sigmasq_alpha ~ inv_gamma(0.001, 0.001); sigmasq_beta ~ inv_gamma(0.001, 0.001); alpha ~ normal(mu_alpha, sigma_alpha); // vectorized beta ~ normal(mu_beta, sigma_beta); // vectorized for (n in 1:N) for (t in 1:T) y[n,t] ~ normal(alpha[n] + beta[n] * (x[t] - xbar), sigma_y); } generated quantities { real alpha0; alpha0 = mu_alpha - xbar * mu_beta; }
rats
Model Summary
Inference for Stan model: rats. 4 chains, each with iter=2000; warmup=1000; thin=1; post-warmup draws per chain=1000, total post-warmup draws=4000. mean se_mean sd 2.5% 25% 50% 75% 97.5% n_eff Rhat alpha[1] 239.93 0.03 2.61 234.75 238.22 239.91 241.71 244.96 6102 1 alpha[2] 247.80 0.04 2.70 242.54 245.97 247.79 249.62 253.07 5745 1 alpha[3] 252.44 0.04 2.60 247.27 250.72 252.46 254.21 257.55 4957 1 alpha[4] 232.56 0.04 2.65 227.48 230.69 232.54 234.34 237.79 5564 1 alpha[5] 231.58 0.03 2.73 226.42 229.71 231.56 233.46 236.86 6405 1 alpha[6] 249.76 0.04 2.71 244.61 247.94 249.74 251.60 255.03 5168 1 alpha[7] 228.66 0.03 2.67 223.37 226.97 228.65 230.47 233.89 6562 1 alpha[8] 248.41 0.03 2.69 243.04 246.62 248.43 250.21 253.52 5921 1 alpha[9] 283.31 0.04 2.70 277.91 281.57 283.37 285.05 288.59 4862 1 alpha[10] 219.31 0.03 2.66 214.14 217.52 219.26 221.14 224.55 5997 1 alpha[11] 258.25 0.04 2.71 252.87 256.39 258.32 260.12 263.41 5403 1 alpha[12] 228.17 0.04 2.63 223.21 226.39 228.14 229.91 233.39 5125 1 alpha[13] 242.39 0.04 2.68 237.08 240.57 242.45 244.22 247.70 5586 1 alpha[14] 268.25 0.04 2.66 262.97 266.43 268.29 270.08 273.34 5341 1 alpha[15] 242.73 0.03 2.65 237.51 241.02 242.71 244.48 247.91 5750 1 alpha[16] 245.35 0.03 2.68 239.99 243.54 245.32 247.15 250.60 6011 1 alpha[17] 232.16 0.04 2.71 226.83 230.31 232.15 233.99 237.51 5920 1 alpha[18] 240.42 0.03 2.64 235.33 238.69 240.43 242.18 245.52 5940 1 alpha[19] 253.77 0.04 2.68 248.45 251.99 253.80 255.54 259.06 5602 1 alpha[20] 241.62 0.03 2.60 236.58 239.91 241.57 243.34 246.78 6264 1 alpha[21] 248.59 0.03 2.70 243.24 246.79 248.56 250.41 253.85 6674 1 alpha[22] 225.31 0.04 2.77 219.92 223.48 225.28 227.18 230.91 6170 1 alpha[23] 228.52 0.03 2.61 223.43 226.79 228.54 230.24 233.62 6656 1 alpha[24] 245.11 0.03 2.62 239.91 243.40 245.14 246.83 250.29 6691 1 alpha[25] 234.44 0.03 2.69 229.26 232.63 234.43 236.23 239.82 6256 1 alpha[26] 253.92 0.04 2.61 248.70 252.16 253.92 255.65 259.09 5479 1 alpha[27] 254.27 0.03 2.57 249.27 252.55 254.25 255.96 259.55 5780 1 alpha[28] 243.01 0.04 2.70 237.55 241.20 243.04 244.87 248.15 5767 1 alpha[29] 217.91 0.03 2.69 212.74 216.09 217.89 219.71 223.13 6317 1 alpha[30] 241.42 0.03 2.61 236.37 239.64 241.41 243.24 246.48 6029 1 beta[1] 6.06 0.00 0.24 5.59 5.91 6.07 6.22 6.53 5584 1 beta[2] 7.05 0.00 0.26 6.55 6.88 7.05 7.22 7.55 4936 1 beta[3] 6.48 0.00 0.24 6.02 6.32 6.48 6.65 6.97 4433 1 beta[4] 5.34 0.00 0.26 4.82 5.17 5.34 5.52 5.84 5458 1 beta[5] 6.57 0.00 0.24 6.09 6.41 6.57 6.73 7.05 5527 1 beta[6] 6.17 0.00 0.24 5.70 6.00 6.17 6.34 6.64 5028 1 beta[7] 5.97 0.00 0.24 5.50 5.81 5.97 6.14 6.44 5714 1 beta[8] 6.42 0.00 0.24 5.95 6.25 6.41 6.59 6.90 5518 1 beta[9] 7.05 0.00 0.25 6.54 6.89 7.05 7.22 7.54 5162 1 beta[10] 5.84 0.00 0.24 5.36 5.68 5.84 6.00 6.31 5171 1 beta[11] 6.80 0.00 0.25 6.31 6.63 6.80 6.97 7.28 5098 1 beta[12] 6.12 0.00 0.24 5.65 5.96 6.11 6.28 6.58 5552 1 beta[13] 6.16 0.00 0.25 5.65 6.01 6.16 6.32 6.66 5429 1 beta[14] 6.69 0.00 0.24 6.22 6.52 6.69 6.85 7.17 5107 1 beta[15] 5.42 0.00 0.25 4.94 5.25 5.41 5.59 5.91 4556 1 beta[16] 5.93 0.00 0.24 5.45 5.77 5.93 6.09 6.39 5506 1 beta[17] 6.28 0.00 0.24 5.82 6.12 6.28 6.44 6.74 5684 1 beta[18] 5.84 0.00 0.24 5.36 5.68 5.83 6.00 6.30 5159 1 beta[19] 6.40 0.00 0.24 5.93 6.23 6.40 6.56 6.85 5036 1 beta[20] 6.05 0.00 0.25 5.56 5.89 6.05 6.22 6.54 6193 1 beta[21] 6.40 0.00 0.24 5.93 6.24 6.40 6.56 6.86 6641 1 beta[22] 5.86 0.00 0.24 5.40 5.69 5.86 6.02 6.31 5890 1 beta[23] 5.75 0.00 0.24 5.27 5.59 5.75 5.91 6.23 6016 1 beta[24] 5.89 0.00 0.24 5.41 5.73 5.89 6.05 6.37 6260 1 beta[25] 6.91 0.00 0.25 6.42 6.74 6.90 7.07 7.40 4974 1 beta[26] 6.54 0.00 0.24 6.06 6.39 6.55 6.70 7.01 5722 1 beta[27] 5.90 0.00 0.24 5.41 5.73 5.90 6.06 6.38 5821 1 beta[28] 5.85 0.00 0.23 5.40 5.69 5.84 6.01 6.31 5740 1 beta[29] 5.68 0.00 0.25 5.20 5.51 5.67 5.84 6.17 5303 1 beta[30] 6.13 0.00 0.23 5.68 5.97 6.12 6.28 6.59 6428 1 mu_alpha 242.47 0.05 2.76 236.95 240.61 242.50 244.38 247.70 3585 1 mu_beta 6.19 0.00 0.11 5.98 6.12 6.19 6.25 6.40 4462 1 sigmasq_y 37.16 0.12 5.69 27.74 33.14 36.56 40.58 50.12 2366 1 sigmasq_alpha 218.39 1.06 63.89 126.08 173.31 208.62 251.30 372.24 3615 1 sigmasq_beta 0.27 0.00 0.10 0.13 0.21 0.26 0.32 0.52 3028 1 sigma_y 6.08 0.01 0.46 5.27 5.76 6.05 6.37 7.08 2370 1 sigma_alpha 14.63 0.03 2.07 11.23 13.16 14.44 15.85 19.29 3919 1 sigma_beta 0.52 0.00 0.09 0.36 0.45 0.51 0.57 0.72 2897 1 alpha0 106.39 0.06 3.60 99.23 104.00 106.44 108.76 113.55 4122 1 lp__ -437.92 0.21 7.04 -453.36 -442.36 -437.34 -432.93 -425.72 1098 1 Samples were drawn using NUTS(diag_e) at Fri Jul 31 07:47:56 2020. For each parameter, n_eff is a crude measure of effective sample size, and Rhat is the potential scale reduction factor on split chains (at convergence, Rhat=1).
We use the butcher package to evaluate the size of each element.
rats%>% attributes()%>% butcher::weigh(units = 'MB') #> # A tibble: 404 x 2 #> object size #> <chr> <dbl> #> 1 stanmodel 5.86 #> 2 .MISC 0.0449 #> 3 sim.samples.alpha[1] 0.0160 #> 4 sim.samples.alpha[2] 0.0160 #> 5 sim.samples.alpha[3] 0.0160 #> 6 sim.samples.alpha[4] 0.0160 #> 7 sim.samples.alpha[5] 0.0160 #> 8 sim.samples.alpha[6] 0.0160 #> 9 sim.samples.alpha[7] 0.0160 #> 10 sim.samples.alpha[8] 0.0160 #> # … with 394 more rows
shredder
can remove three elements in the stanfit
object
.MISC
$summarysummary <- function(x){ y <- x%>% attributes()%>% butcher::weigh() s <- y$size data.frame(min = min(s), max = max(s), mean = mean(s), sd = sd(s), sum = sum(s)) }
butcher_rats <- rats%>% summary() butcher_fit_instance <- rats%>% stan_axe(what = 'fit_instance')%>% summary() butcher_stanmodel <- rats%>% stan_axe(what = 'fit_instance')%>% stan_axe(what = 'stanmodel')%>% summary() butcher_summary <- rats%>% stan_axe(what = 'fit_instance')%>% stan_axe(what = 'stanmodel')%>% stan_axe(what = 'summary')%>% summary() butcher_params <- rats%>% stan_axe(what = 'fit_instance')%>% stan_axe(what = 'stanmodel')%>% stan_axe(what = 'summary')%>% stan_select(alpha)%>% summary()
tbl <- purrr::map_df( list('full' = butcher_rats, 'fit_instance' = butcher_fit_instance, 'stanmodel' = butcher_stanmodel, 'summary' = butcher_summary, 'param' = butcher_params), identity,.id='axe')
knitr::kable(tbl)
axe | min | max | mean | sd | sum |
---|---|---|---|---|---|
full | 4.8e-05 | 5.860896 | 0.0258238 | 0.2911213 | 10.432832 |
fit_instance | 4.8e-05 | 5.860896 | 0.0258238 | 0.2911213 | 10.432832 |
stanmodel | 4.8e-05 | 0.044880 | 0.0113448 | 0.0074566 | 4.571936 |
summary | 4.8e-05 | 0.016048 | 0.0112342 | 0.0072865 | 4.527392 |
param | 5.6e-05 | 0.016048 | 0.0108903 | 0.0073964 | 2.014712 |