Code
library(tidyverse)
library(corrr)
library(showtext)
library(gt)
library(formattable)
library(patchwork)
library(psych)
requireNamespace("bit64")Code
pivot_wider_indices <- function(data, name_value = "test", always_suffix = FALSE) {
data |>
group_by(game_name) |>
mutate(n_index = n_distinct(index_name)) |>
ungroup() |>
mutate(
game_index = if (always_suffix) {
str_c(game_name, index_name, sep = ".")
} else {
if_else(
n_index == 1,
game_name,
str_c(game_name, index_name, sep = ".")
)
}
) |>
pivot_wider(
id_cols = user_id,
names_from = game_index,
values_from = .data[[name_value]]
)
}
summary_data <- function(data, name_value = "test") {
pivot_wider_indices(data, name_value, always_suffix = TRUE) |>
select(-user_id) |>
psych::describe() |>
as_tibble(rownames = "game_index_name") |>
select(game_index_name, n, mean, sd, min, max, skew, kurtosis) |>
separate(game_index_name, c("game_name", "index_name"), sep = "\\.") |>
mutate(
skew_status = case_when(
abs(skew) > 1 ~ "highly",
abs(skew) > 0.5 ~ "moderately",
TRUE ~ "acceptable"
)
)
}
show_normality <- function(data, game_name, index_name, skew, ...,
name_value = "test") {
p_hist <- ggplot(data, aes_string(name_value)) +
geom_histogram(bins = 30) +
labs(x = "Raw Score", y = "Count") +
theme_bw()
p_qq <- ggplot(data, aes_string(sample = name_value)) +
geom_qq() +
geom_qq_line() +
labs(x = "Expected Quantile", y = "Observed Quantile") +
theme_bw()
wrap_elements(grid::textGrob(
str_glue("{game_name}\n{index_name}\nskewness = {round(skew, 2)}"),
rot = 90
)) + p_hist + p_qq +
plot_layout(widths = c(0.1, 0.45, 0.45))
}
transform_indices <- function(x, game_name, index_name) {
if (game_name == "小狗回家" & index_name == "mean_score") {
return(-log10(1 - x))
}
if (game_name == "各得其所" & index_name == "mrt_init") {
return(log10(x))
}
if (game_name == "数字推理") {
return(sqrt(x))
}
if (game_name == "塔罗牌") {
return(-sqrt(max(x, na.rm = TRUE) - x))
}
if (game_name == "人工语言-中级") {
return(-sqrt(max(x, na.rm = TRUE) - x))
}
if (game_name == "语义判断") {
return(-log10(max(x, na.rm = TRUE) + 1 - x))
}
if (game_name == "数感") {
return(log10(x))
}
if (game_name == "时长分辨") {
return(log10(x))
}
if (game_name == "节奏感知") {
return(sqrt(x))
}
return(x)
}
calc_test_retest <- function(data,
name_test = "test",
name_retest = "retest") {
data_no_missing <- data |>
select(all_of(c(name_test, name_retest))) |>
drop_na()
outliers_result <- data_no_missing |>
performance::check_outliers(method = "mahalanobis_robust")
data_no_outlier <- data_no_missing |>
slice(-which(outliers_result))
icc_result <- ICC(data_no_outlier)
data.frame(
n_obs = icc_result$n.obs,
icc = icc_result$results$ICC[[2]],
r = cor(data_no_outlier[[name_test]], data_no_outlier[[name_retest]])
)
}
vis_cor <- function(x, ...) {
x |>
correlate(...) |>
autoplot(
triangular = "full",
low = "blue",
high = "red"
) +
theme(aspect.ratio = 1)
}
format_n_factors <- function(n_factor_result) {
checks <- c("BIC", "SABIC", "eBIC", "MAP")
n_factor_result$vss.stats |>
as_tibble() |>
select(BIC, SABIC, eBIC) |>
mutate(n_factor = seq_len(n()), .before = 1L) |>
add_column(MAP = n_factor_result$map) |>
gt() |>
fmt_number(
-c(n_factor, MAP),
decimals = 0
) |>
fmt_scientific(MAP) |>
tab_style(
style = cell_text(weight = "bold"),
locations = map(
checks,
~ cells_body(
columns = all_of(.x),
rows = !!sym(.x) == min(!!sym(.x))
)
)
) |>
as_raw_html()
}
plotly_efa <- function(model) {
heatmaply::heatmaply_cor(
t(unclass(model$loadings)),
dendrogram = "column",
k_col = NA,
label_format_fun = function(...) round(..., digits = 2),
margin = c(50, 50, 50, 0)
)
}Code
targets::tar_load(users)
targets::tar_load(indices_selection)
targets::tar_load(indices_clean)
dim_order <- c(
"attention", "multitask", "switching", "inhibition",
"reasoning", "complex span", "working memory", "short term memory",
"long term memory", "probability learning",
"speeded IP", "strategic IP",
"perception", "math", "language"
)
indices_outlier_info <- indices_clean |>
full_join(
indices_selection,
by = c("game_name", "game_name_abbr", "index_name")
) |>
filter(check_result %in% c("target", "target-alt2")) |>
mutate(score = if_else(reversed == "yes", -test, test)) |>
arrange(factor(dimension, dim_order)) |>
group_by(game_name, index_name) |>
mutate(
is_outlier = unclass(performance::check_outliers(test, method = "iqr"))
) |>
ungroup()
indices <- indices_outlier_info |>
filter(!is_outlier) |>
select(-is_outlier)
indices_stats <- summary_data(indices)Data Cleaning Summary
Careless responses will ruin data, especially when there are 10% or more participants respond carelessly (Woods 2006). From Figure 1, it shows that there were two tasks in which more than 10% of participants performed badly (labelled 'unmotivated' in the figure). These two games are known to be difficult. So we can conclude that participants were all motivated to fulfill all the tasks, there is no need to screen unmotivated participants.
Code
targets::tar_load(user_resp_check)
user_resp_check |>
group_by(game_name, is_motivated) |>
summarise(n = n(), .groups = "drop_last") |>
mutate(prop = n / sum(n)) |>
ungroup() |>
arrange(desc(prop)) |>
mutate(
game_name = as_factor(game_name),
is_motivated = factor(is_motivated, c(FALSE, TRUE), c("Unmotivated", "Motivated"))
) |>
ggplot(aes(game_name, n, fill = is_motivated)) +
geom_col() +
geom_text(
aes(label = scales::label_percent(accuracy = 0.1)(prop)),
position = position_stack(vjust = 0.5)
) +
scale_fill_brewer(palette = "Paired") +
coord_flip() +
ggpubr::theme_pubclean(flip = TRUE) +
theme(
axis.title.x = element_blank(),
axis.title.y = element_blank(),
legend.title = element_blank()
)
Another issue is outliers. We screened out one-dimension outliers using Tukey’s boxplot method, with a coefficient of 1.5, i.e., participants perform out of 1.5 inter-quantile range from the first and third quartiles are screened out. Figure 2 shows a summary of removed participants for all tasks, in which the task ‘节奏感知’ stood out, with 35 participants removed.
Code
indices_outlier_info |>
group_by(game_name, index_name) |>
summarise(n_outlier = sum(is_outlier, na.rm = TRUE), .groups = "drop") |>
ggplot(aes(n_outlier)) +
geom_histogram(fill = "white", color = "black", bins = 30) +
theme_bw() +
labs(x = "Count of Outliers", y = "Count of Task Indices")
Data Checking on Normality
It is necessary to check normality before performing exploratory analysis.
Original Form
Code
indices |>
ggplot(aes(test)) +
geom_histogram(bins = 30) +
facet_wrap(vars(game_name, index_name), scales = "free", ncol = 5) +
labs(x = "Raw Score", y = "Count") +
theme_bw()
Code
indices |>
ggplot(aes(sample = test)) +
geom_qq() +
geom_qq_line() +
facet_wrap(vars(game_name, index_name), scales = "free", ncol = 5) +
labs(x = "Expected Quantile", y = "Observed Quantile") +
theme_bw()
According to a classic book (Kline 2016), critical value for absolute skewness is 3, and critical value for absolute kurtosis is 10; thus, these indices are all not too apart from normal distribution. But see Figure 3, we detect some heavily skewed indices. Just as a classic book (Bulmer 1979) said, critical value for skewness is 0.5 (moderately skew) and 1 (highly skew).
Code
indices_stats |>
mutate(across(mean:kurtosis, ~ digits(., 3))) |>
formattable() |>
as.datatable(
caption = "Descriptive Statistics of All Indices"
)Transformed
Here we decide to convert the moderately and highly indices based on the conversion suggested by Howell (2013).
Code
indices |>
inner_join(
indices_stats |>
select(game_name, index_name, skew, skew_status) |>
filter(skew_status != "acceptable"),
by = c("game_name", "index_name")
) |>
group_nest(game_name, index_name, skew, skew_status) |>
pmap(show_normality) |>
wrap_plots(ncol = 1)
From Figure 5, it might be a good choice to transform indices with an absolute skewness value larger than 0.5, and logarithmic and square root transformations are used.
Code
game_needs_trans <- indices_stats |>
filter(abs(skew) > 0.5)
indices_trans <- indices |>
semi_join(game_needs_trans, by = c("game_name", "index_name")) |>
group_by(game_name, index_name) |>
group_modify(
~ .x |>
mutate(
across(
c(test, retest),
transform_indices,
game_name = .y$game_name,
index_name = .y$index_name
)
)
) |>
ungroup() |>
mutate(score = if_else(reversed == "yes", -test, test))
indices_stats_trans <- summary_data(indices_trans)
indices_pooled <- bind_rows(
origin = indices,
trans = indices_trans,
.id = "index_type"
) |>
left_join(
game_needs_trans |>
add_column(need_trans = TRUE) |>
select(game_name, index_name, need_trans),
by = c("game_name", "index_name")
) |>
mutate(need_trans = coalesce(need_trans, FALSE))Code
indices_trans |>
inner_join(indices_stats_trans, by = c("game_name", "index_name")) |>
group_nest(game_name, index_name, skew, skew_status) |>
pmap(show_normality) |>
wrap_plots(ncol = 1)
From Figure 6, we know these indices now perform very well on normality. The transformations seem to be useful.
Data Checking on Reliability
Code
test_retest <- indices_pooled |>
group_by(game_name, index_name, dimension, index_type, need_trans) |>
group_modify(~ calc_test_retest(.)) |>
ungroup()
format_na <- function(str, x) {
ifelse(is.na(x), "-", str)
}
test_retest |>
pivot_wider(
id_cols = c(game_name, index_name),
names_from = index_type,
values_from = c(n_obs, icc, r),
names_vary = "slowest"
) |>
mutate(
across(
starts_with(c("icc", "r")),
~ digits(., 2, postproc = format_na)
),
n_obs_trans = digits(n_obs_trans, 0, postproc = format_na)
) |>
formattable(
list(
formattable::area(col = c(icc_origin, icc_trans)) ~
color_tile("white", "pink", na.rm = TRUE),
game_name = formatter(
"span",
style = ~ if_else(
coalesce(icc_trans, icc_origin) < 0.5,
"background:grey",
NA_character_
)
)
)
) |>
as.datatable()From the above table, it shows that the transformations are successful in elevating test-retest reliability. Kline stated in their book (2013): “If \(r_{xx} < .50\), then most of the total variance is due to measurement error. Indicators with such low score reliabilities should be excluded from the analysis.” We should remove those indices with test-retest reliability less than 0.5 (grey background cells) according to this quote. The removed indicators are:
Code
indicators_remove <- test_retest |>
filter(!need_trans | (index_type == "trans" & need_trans)) |>
filter(icc < 0.5) |>
select(game_name, index_name, dimension, icc)
indicators_remove |>
knitr::kable(
digits = 2,
caption = "Removed Indicators",
row.names = TRUE
)| game_name | index_name | dimension | icc | |
|---|---|---|---|---|
| 1 | 按图索骥 | mean_log_err_both | perception | 0.49 |
| 2 | 变色魔块PRO | ssrt | inhibition | 0.40 |
| 3 | 察颜观色PRO | switch_cost_mrt | switching | 0.45 |
| 4 | 打靶场 | nc | complex span | 0.45 |
| 5 | 各得其所 | prop_perfect | reasoning | 0.33 |
| 6 | 候鸟迁徙PRO | switch_cost_mrt | switching | 0.23 |
| 7 | 节奏感知 | thresh_last_block | perception | 0.35 |
| 8 | 卡片分类PRO | switch_cost_mrt | switching | 0.42 |
| 9 | 密码箱 | nc | working memory | 0.46 |
| 10 | 魔术师终极 | dprime | working memory | 0.41 |
| 11 | 强化学习 | pc_test | probability learning | 0.29 |
| 12 | 人工语言-中级 | nc | associative learning | 0.49 |
| 13 | 数字卡片PRO | dprime | working memory | 0.47 |
| 14 | 图形推理 | nc | reasoning | 0.50 |
| 15 | 图形折叠 | nc | reasoning | 0.46 |
| 16 | 文字推理 | nc | reasoning | 0.30 |
| 17 | 我是大厨 | score_total | multitask | 0.22 |
| 18 | 雪花收藏家 | nc_cor | attention | 0.48 |
| 19 | 言语记忆 | fm_dprime | long term memory | 0.04 |
| 20 | 远距离联想 | nc | reasoning | 0.35 |
| 21 | 捉虫高级简版 | mrt | speeded IP | 0.49 |
Code
indices_pooled |>
filter(!need_trans | (index_type == "trans" & need_trans)) |>
semi_join(indicators_remove, by = c("game_name", "index_name", "dimension")) |>
drop_na() |>
ggplot(aes(test, retest)) +
geom_point() +
ggpmisc::stat_poly_line() +
ggpmisc::stat_correlation(small.r = TRUE) +
geom_abline(slope = 1, intercept = 0, linetype = "dotted", color = "red") +
facet_wrap(vars(game_name, index_name), scales = "free", ncol = 3) +
labs(x = "Test", y = "Retest") +
theme_bw() +
theme(aspect.ratio = 1)
careless_suspect <- targets::tar_read(data_parsed_NVRB) |>
unnest(raw_parsed) |>
group_by(user_id) |>
filter(mean(rt) < 2000) |>
ungroup() |>
distinct(user_id)
indices_pooled |>
filter(!need_trans | (index_type == "trans" & need_trans)) |>
semi_join(indicators_remove, by = c("game_name", "index_name", "dimension")) |>
anti_join(careless_suspect, by = "user_id") |>
drop_na() |>
ggplot(aes(test, retest)) +
geom_point() +
ggpmisc::stat_poly_line() +
ggpmisc::stat_correlation(small.r = TRUE) +
geom_abline(slope = 1, intercept = 0, linetype = "dotted", color = "red") +
facet_wrap(vars(game_name, index_name), scales = "free", ncol = 3) +
labs(x = "Test", y = "Retest") +
theme_bw() +
theme(aspect.ratio = 1)
indices_pooled |>
filter(!need_trans | (index_type == "trans" & need_trans)) |>
semi_join(indicators_remove, by = c("game_name", "index_name", "dimension")) |>
anti_join(careless_suspect, by = "user_id") |>
drop_na() |>
group_by(game_name, index_name) |>
mutate(
performance::check_outliers(
tibble(test, retest),
method = "mahalanobis_robust"
) |>
as_tibble()
) |>
ungroup() |>
filter(!Outlier) |>
ggplot(aes(test, retest)) +
geom_point() +
ggpmisc::stat_poly_line() +
ggpmisc::stat_correlation(small.r = TRUE) +
geom_abline(slope = 1, intercept = 0, linetype = "dotted", color = "red") +
facet_wrap(vars(game_name, index_name), scales = "free", ncol = 3) +
labs(x = "Test", y = "Retest") +
theme_bw() +
theme(aspect.ratio = 1)
Note there is a very special case, i.e. “我是大厨”, whose pearson correlation coefficient is substantially larger than ICC value. Further analysis (see Figure 10) shows that participants improve more if they perform worse in the first test. In this case, it makes us doubt whether intraclass correlation (ICC) is suitable for cognitive ability tests for there is always practice effect (they do no really use parallel versions).
Code
indices_pooled |>
filter(game_name == "我是大厨") |>
ggplot(aes(test, retest)) +
geom_point() +
geom_smooth(method = "lm", formula = y ~ x) +
geom_abline(
slope = 1,
intercept = 0,
linetype = "dotted",
color = "lightblue"
) +
theme_bw() +
coord_fixed() +
labs(x = "Test Phase", y = "Retest Phase")
But see Figure 11, these two reliability indices are closely correlated with each other.
Code
test_retest |>
ggplot(aes(icc, r)) +
geom_point() +
stat_smooth(method = "lm", formula = y ~ x) +
ggpmisc::stat_correlation(small.r = TRUE) +
geom_abline(
slope = 1,
intercept = 0,
linetype = "dotted",
color = "lightblue"
) +
theme_bw() +
labs(x = "ICC", y = "Pearson's Coefficient") +
coord_fixed()
Data Checking on Correlation Matrix
Code
indices_wider <- indices_pooled |>
anti_join(indicators_remove, by = c("game_name", "index_name")) |>
filter(!need_trans | (index_type == "trans" & need_trans)) |>
pivot_wider_indices(
name_value = "score",
always_suffix = TRUE
) |>
left_join(
users |>
mutate(school = str_remove(school, "认知实验")) |>
select(user_id, school),
by = "user_id"
)
indices_wider_bare <- select(indices_wider, -user_id, -school)
indices_wider_bare_sic <- indices_wider |>
filter(school == "四川师范大学") |>
select(-user_id, -school)
indices_wider_bare_bj <- indices_wider |>
filter(school != "四川师范大学") |>
select(-user_id, -school)Code
naniar::vis_miss(indices_wider_bare) +
scale_y_continuous(expand = c(0, 0), trans = "reverse") +
theme(axis.text.x = element_text(angle = 90))
Code
vis_cor(indices_wider_bare, quiet = TRUE)
vis_cor(indices_wider_bare_sic, quiet = TRUE)
vis_cor(indices_wider_bare_bj, quiet = TRUE)
The correlation matrix of Beijing Data set is not positive definite, thus not appropriate for further analysis. Further check on the data shows there are only 21 complete cases, which corrupts the data set. But the whole data set and Sichuan sub data set are all suitable for further analysis.
Bartlett’s Test
Code
list(
Whole = indices_wider_bare,
`Sichuan Dataset` = indices_wider_bare_sic
) |>
map(
~ psych::cortest.bartlett(.) |>
as_tibble()
) |>
bind_rows(.id = "Data Source") |>
knitr::kable(digits = 2)| Data Source | chisq | p.value | df |
|---|---|---|---|
| Whole | 5414.76 | 0 | 780 |
| Sichuan Dataset | 3428.56 | 0 | 780 |
Sampling Adequacy
KMO test: this test will give us an index of how well the matrix is suited for factor analysis. The criteria suggested by Kaiser (1974):
Code
kmo_result <- psych::KMO(indices_wider_bare)
kmo_result_sic <- psych::KMO(indices_wider_bare_sic)
bind_rows(
whole = kmo_result$MSAi |>
enframe(name = "game_index_name", value = "MSA"),
sichuan = kmo_result_sic$MSAi |>
enframe(name = "game_index_name", value = "MSA"),
.id = "src"
) |>
separate(game_index_name, c("game_name", "index_name"), sep = "\\.") |>
mutate(MSA = digits(MSA, 2)) |>
pivot_wider(
names_from = src,
values_from = MSA,
names_prefix = "MSA_"
) |>
formattable() |>
as.datatable(caption = "KMO Test Result")For the whole data set, the overall MSA value is 0.86. However, there are 0 indices with MSA value lower than 0.6, which is generally acceptable.
For the Sichuan data set, the overall MSA value is 0.78. However, there are 2 indices with MSA value lower than 0.6, which is generally acceptable.
Exploratory Factor Analysis
Determining Number of Factors
Code
n_factor_whole <- psych::nfactors(indices_wider_bare)Code
n_factor_sic <- psych::nfactors(indices_wider_bare_sic)Code
format_n_factors(n_factor_whole)
format_n_factors(n_factor_sic)Table 2: Number of Factors Tests
From Table 2, we noticed inconsistencies. So we try factor numbers as 9, 6 and 3.
Results of 9 factors
Code
fitted <- fa(indices_wider_bare, 9)
plotly_efa(fitted)Code
parameters::model_parameters(fitted, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR3 | MR6 | MR7 | MR5 | MR2 | MR4 | MR8 | MR1 | MR9 | Complexity | Uniqueness
------------------------------------------------------------------------------------------------------------------------
速算师（中级）.nc | 0.74 | | | | | | | | | 1.22 | 0.30
专注大师_中级.nc | 0.71 | | | | | | | | | 1.11 | 0.42
声调判断.nc | 0.45 | | | | | | | | | 2.52 | 0.62
快速归类PRO.mrt | 0.34 | | | | | | | | | 3.51 | 0.54
多彩文字PRO.cong_eff_mrt | 0.29 | | | | | | | | | 2.93 | 0.82
数字推理.nc | 0.28 | | | | | | | | | 4.60 | 0.73
位置记忆PRO.nc | | 0.70 | | | | | | | | 1.07 | 0.42
蝴蝶照相机.nc | | 0.64 | | | | | | | | 1.10 | 0.46
萤火虫PRO.nc | | 0.48 | | | | | | | | 1.51 | 0.70
宇宙黑洞A.nc | | 0.33 | | | | | | | | 2.65 | 0.71
格子卡片.dprime | | | 0.91 | | | | | | | 1.01 | 0.18
文字卡片.dprime | | | 0.58 | | | | | | | 1.24 | 0.55
塔罗牌.nc | | | 0.19 | | | | | | | 5.13 | 0.77
幸运小球PRO.nc | | | | 0.64 | | | | | | 1.22 | 0.46
顺背数PRO.nc | | | | 0.52 | | | | | | 1.37 | 0.61
欢乐餐厅PRO.nc | | | | 0.46 | | | | | | 2.54 | 0.63
过目不忘PRO.nc | | | | 0.38 | | | | | | 3.59 | 0.61
小狗回家.mean_score | | | | | 0.67 | | | | | 1.19 | 0.42
一心二用PRO.nc | | | | | 0.44 | | | | | 2.37 | 0.55
连点成画PRO.nc | | | | | 0.38 | | | | | 3.05 | 0.62
太空飞船PRO.cong_eff_mrt | | | | | -0.35 | | | | | 1.79 | 0.86
超级秒表PRO.mrt | | | | | 0.24 | | | | | 4.48 | 0.75
变戏法.nc | | | | | 0.23 | | | | | 5.34 | 0.68
捉虫高级简版.dprime | | | | | | 0.86 | | | | 1.02 | 0.28
捉虫高级简版.rtsd | | | | | | 0.49 | | | | 2.19 | 0.59
连续再认PRO.dprime | | | | | | | 0.43 | | | 1.98 | 0.68
小狗回家.dprime | | | | | | | 0.40 | | | 2.77 | 0.57
语义判断.nc | | | | | | | 0.35 | | | 2.55 | 0.63
时间顺序判断.thresh_last_block | | | | | | | 0.32 | | | 3.43 | 0.66
图片记忆.bps_score | | | | | | | 0.27 | | | 5.09 | 0.61
三维心理旋转测试.nc | | | | | | | | 0.56 | | 1.24 | 0.57
登陆月球（中级）.mean_log_err | | | | | | | | 0.34 | | 2.41 | 0.69
美术卡片.dprime | | | | | | | | 0.28 | | 4.95 | 0.53
候鸟迁徙PRO.cong_eff_mrt | | | | | | | | 0.24 | | 3.28 | 0.77
数感.w | | | | | | | | 0.21 | | 5.81 | 0.74
时长分辨.thresh_last_block | | | | | | | | 0.20 | | 4.25 | 0.83
火眼金睛.nc | | | | | | | | | 0.54 | 1.24 | 0.58
舒尔特方格（中级）.nc_cor | | | | | | | | | 0.31 | 3.83 | 0.66
各得其所.mrt_init | | | | | | | | | 0.30 | 4.31 | 0.73
方向临摹.mean_log_err | | | | | | | | | 0.24 | 5.33 | 0.78
The 9 latent factors (oblimin rotation) accounted for 39.24% of the total variance of the original data (MR3 = 5.84%, MR6 = 5.31%, MR7 = 5.01%, MR5 = 4.35%, MR2 = 4.07%, MR4 = 3.81%, MR8 = 3.77%, MR1 = 3.70%, MR9 = 3.39%).Code
fitted_sic <- fa(indices_wider_bare_sic, 9)
plotly_efa(fitted_sic)Code
parameters::model_parameters(fitted_sic, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR3 | MR2 | MR4 | MR6 | MR9 | MR1 | MR5 | MR7 | MR8 | Complexity | Uniqueness
------------------------------------------------------------------------------------------------------------------------
专注大师_中级.nc | 0.74 | | | | | | | | | 1.07 | 0.41
速算师（中级）.nc | 0.65 | | | | | | | | | 1.33 | 0.45
声调判断.nc | 0.53 | | | | | | | | | 1.62 | 0.63
快速归类PRO.mrt | 0.42 | | | | | | | | | 2.96 | 0.48
时间顺序判断.thresh_last_block | 0.41 | | | | | | | | | 3.45 | 0.57
语义判断.nc | 0.34 | | | | | | | | | 3.43 | 0.66
多彩文字PRO.cong_eff_mrt | 0.27 | | | | | | | | | 2.74 | 0.88
时长分辨.thresh_last_block | 0.24 | | | | | | | | | 4.21 | 0.82
小狗回家.mean_score | | 0.67 | | | | | | | | 1.42 | 0.40
连点成画PRO.nc | | 0.56 | | | | | | | | 1.38 | 0.64
一心二用PRO.nc | | 0.48 | | | | | | | | 1.94 | 0.57
超级秒表PRO.mrt | | 0.36 | | | | | | | | 3.30 | 0.67
变戏法.nc | | 0.34 | | | | | | | | 2.73 | 0.67
各得其所.mrt_init | | 0.33 | | | | | | | | 2.66 | 0.82
太空飞船PRO.cong_eff_mrt | | -0.25 | | | | | | | | 3.39 | 0.88
位置记忆PRO.nc | | | 0.76 | | | | | | | 1.03 | 0.35
蝴蝶照相机.nc | | | 0.63 | | | | | | | 1.20 | 0.49
萤火虫PRO.nc | | | 0.41 | | | | | | | 2.28 | 0.73
宇宙黑洞A.nc | | | 0.33 | | | | | | | 3.51 | 0.63
文字卡片.dprime | | | | 0.76 | | | | | | 1.06 | 0.42
格子卡片.dprime | | | | 0.75 | | | | | | 1.16 | 0.31
三维心理旋转测试.nc | | | | | 0.57 | | | | | 1.29 | 0.60
美术卡片.dprime | | | | | 0.33 | | | | | 4.17 | 0.58
数感.w | | | | | 0.31 | | | | | 3.24 | 0.77
登陆月球（中级）.mean_log_err | | | | | 0.31 | | | | | 2.16 | 0.81
数字推理.nc | | | | | 0.30 | | | | | 5.44 | 0.67
火眼金睛.nc | | | | | 0.30 | | | | | 3.96 | 0.65
候鸟迁徙PRO.cong_eff_mrt | | | | | 0.22 | | | | | 4.55 | 0.75
连续再认PRO.dprime | | | | | | 0.48 | | | | 2.01 | 0.61
图片记忆.bps_score | | | | | | 0.42 | | | | 2.88 | 0.57
小狗回家.dprime | | | | | | 0.42 | | | | 2.85 | 0.53
塔罗牌.nc | | | | | | 0.27 | | | | 3.42 | 0.76
幸运小球PRO.nc | | | | | | | 0.61 | | | 1.19 | 0.54
顺背数PRO.nc | | | | | | | 0.52 | | | 1.46 | 0.66
过目不忘PRO.nc | | | | | | | 0.47 | | | 2.30 | 0.59
欢乐餐厅PRO.nc | | | | | | | 0.39 | | | 3.73 | 0.50
捉虫高级简版.dprime | | | | | | | | 0.69 | | 1.20 | 0.43
捉虫高级简版.rtsd | | | | | | | | 0.46 | | 2.25 | 0.62
方向临摹.mean_log_err | | | | | | | | 0.28 | | 4.85 | 0.77
舒尔特方格（中级）.nc_cor | | | | | | | | | 0.61 | 1.78 | 0.42
The 9 latent factors (oblimin rotation) accounted for 39.16% of the total variance of the original data (MR3 = 6.21%, MR2 = 5.07%, MR4 = 5.01%, MR6 = 4.42%, MR9 = 4.16%, MR1 = 4.05%, MR5 = 3.83%, MR7 = 3.69%, MR8 = 2.73%).These two models have the least sample size adjusted BIC, but it captures many trivial factors with two to three task indices. Factors need at least three manifest variables to be stable or meaningful. But see Figure 13, these factors are a reflection of those task pairs with a rather high correlation (about 0.5) with each other, both of which have relatively lower correlations with other tasks.
Results of 6 factors
Code
fitted <- fa(indices_wider_bare, 6)
plotly_efa(fitted)Code
parameters::model_parameters(fitted, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR1 | MR2 | MR3 | MR4 | MR5 | MR6 | Complexity | Uniqueness
---------------------------------------------------------------------------------------------------
三维心理旋转测试.nc | 0.53 | | | | | | 1.84 | 0.63
美术卡片.dprime | 0.52 | | | | | | 1.45 | 0.54
图片记忆.bps_score | 0.50 | | | | | | 1.76 | 0.61
格子卡片.dprime | 0.44 | | | | | | 2.23 | 0.55
数感.w | 0.40 | | | | | | 1.32 | 0.76
塔罗牌.nc | 0.37 | | | | | | 1.80 | 0.76
连续再认PRO.dprime | 0.33 | | | | | | 2.86 | 0.73
登陆月球（中级）.mean_log_err | 0.32 | | | | | | 2.13 | 0.75
时间顺序判断.thresh_last_block | 0.32 | | | | | | 3.63 | 0.68
候鸟迁徙PRO.cong_eff_mrt | 0.30 | | | | | | 2.22 | 0.78
文字卡片.dprime | 0.28 | | | | | | 3.63 | 0.66
时长分辨.thresh_last_block | 0.22 | | | | | | 3.56 | 0.84
小狗回家.mean_score | | 0.60 | | | | | 1.23 | 0.55
一心二用PRO.nc | | 0.56 | | | | | 1.50 | 0.54
连点成画PRO.nc | | 0.46 | | | | | 1.79 | 0.66
快速归类PRO.mrt | | 0.45 | | | | | 2.20 | 0.56
火眼金睛.nc | | 0.36 | | | | | 2.22 | 0.71
各得其所.mrt_init | | 0.33 | | | | | 2.34 | 0.80
太空飞船PRO.cong_eff_mrt | | -0.31 | | | | | 2.69 | 0.87
超级秒表PRO.mrt | | 0.27 | | | | | 3.68 | 0.76
变戏法.nc | | 0.27 | | | | | 3.68 | 0.70
速算师（中级）.nc | | | 0.77 | | | | 1.12 | 0.29
专注大师_中级.nc | | | 0.69 | | | | 1.13 | 0.43
声调判断.nc | | | 0.39 | | | | 3.23 | 0.64
多彩文字PRO.cong_eff_mrt | | | 0.30 | | | | 2.59 | 0.83
数字推理.nc | | | 0.26 | | | | 3.58 | 0.77
语义判断.nc | | | 0.24 | | | | 3.93 | 0.68
位置记忆PRO.nc | | | | 0.66 | | | 1.11 | 0.45
蝴蝶照相机.nc | | | | 0.61 | | | 1.14 | 0.48
萤火虫PRO.nc | | | | 0.49 | | | 1.59 | 0.70
宇宙黑洞A.nc | | | | 0.36 | | | 2.42 | 0.70
舒尔特方格（中级）.nc_cor | | | | 0.32 | | | 2.13 | 0.73
幸运小球PRO.nc | | | | | 0.70 | | 1.05 | 0.46
顺背数PRO.nc | | | | | 0.57 | | 1.07 | 0.62
欢乐餐厅PRO.nc | | | | | 0.38 | | 2.05 | 0.76
过目不忘PRO.nc | | | | | 0.32 | | 2.73 | 0.69
方向临摹.mean_log_err | | | | | 0.22 | | 3.84 | 0.84
捉虫高级简版.rtsd | | | | | | 0.61 | 1.11 | 0.59
捉虫高级简版.dprime | | | | | | 0.45 | 1.61 | 0.67
小狗回家.dprime | | | | | | 0.26 | 3.92 | 0.66
The 6 latent factors (oblimin rotation) accounted for 33.93% of the total variance of the original data (MR1 = 7.63%, MR2 = 5.81%, MR3 = 5.78%, MR4 = 5.49%, MR5 = 5.19%, MR6 = 4.02%).Code
fitted_sic <- fa(indices_wider_bare_sic, 6)
plotly_efa(fitted_sic)Code
parameters::model_parameters(fitted_sic, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR1 | MR3 | MR2 | MR4 | MR5 | MR6 | Complexity | Uniqueness
---------------------------------------------------------------------------------------------------
图片记忆.bps_score | 0.53 | | | | | | 1.74 | 0.59
美术卡片.dprime | 0.47 | | | | | | 1.88 | 0.60
数感.w | 0.43 | | | | | | 1.23 | 0.80
语义判断.nc | 0.40 | | | | | | 2.15 | 0.66
塔罗牌.nc | 0.37 | | | | | | 1.48 | 0.78
三维心理旋转测试.nc | 0.34 | | | | | | 2.73 | 0.75
连续再认PRO.dprime | 0.32 | | | | | | 2.63 | 0.74
候鸟迁徙PRO.cong_eff_mrt | 0.30 | | | | | | 2.10 | 0.77
时长分辨.thresh_last_block | 0.25 | | | | | | 2.44 | 0.84
登陆月球（中级）.mean_log_err | 0.19 | | | | | | 3.58 | 0.86
专注大师_中级.nc | | 0.77 | | | | | 1.01 | 0.40
速算师（中级）.nc | | 0.63 | | | | | 1.24 | 0.50
声调判断.nc | | 0.55 | | | | | 1.59 | 0.63
时间顺序判断.thresh_last_block | | 0.38 | | | | | 2.50 | 0.62
快速归类PRO.mrt | | 0.38 | | | | | 2.88 | 0.55
捉虫高级简版.rtsd | | 0.33 | | | | | 2.49 | 0.76
多彩文字PRO.cong_eff_mrt | | 0.24 | | | | | 3.18 | 0.88
小狗回家.mean_score | | | 0.70 | | | | 1.05 | 0.49
连点成画PRO.nc | | | 0.57 | | | | 1.33 | 0.67
一心二用PRO.nc | | | 0.54 | | | | 1.35 | 0.58
变戏法.nc | | | 0.40 | | | | 2.55 | 0.67
超级秒表PRO.mrt | | | 0.37 | | | | 3.10 | 0.66
火眼金睛.nc | | | 0.36 | | | | 2.21 | 0.71
各得其所.mrt_init | | | 0.34 | | | | 2.28 | 0.84
太空飞船PRO.cong_eff_mrt | | | -0.26 | | | | 2.84 | 0.88
过目不忘PRO.nc | | | | 0.62 | | | 1.29 | 0.57
顺背数PRO.nc | | | | 0.52 | | | 1.28 | 0.71
幸运小球PRO.nc | | | | 0.51 | | | 1.21 | 0.67
欢乐餐厅PRO.nc | | | | 0.40 | | | 1.92 | 0.75
小狗回家.dprime | | | | 0.36 | | | 2.60 | 0.62
方向临摹.mean_log_err | | | | 0.20 | | | 4.38 | 0.85
格子卡片.dprime | | | | | 0.80 | | 1.08 | 0.30
文字卡片.dprime | | | | | 0.71 | | 1.10 | 0.48
数字推理.nc | | | | | 0.31 | | 2.13 | 0.82
捉虫高级简版.dprime | | | | | 0.29 | | 3.65 | 0.69
舒尔特方格（中级）.nc_cor | | | | | | 0.50 | 1.41 | 0.63
位置记忆PRO.nc | | | | | | 0.39 | 2.90 | 0.55
蝴蝶照相机.nc | | | | | | 0.38 | 2.61 | 0.58
宇宙黑洞A.nc | | | | | | 0.36 | 2.73 | 0.63
萤火虫PRO.nc | | | | | | 0.34 | 2.31 | 0.77
The 6 latent factors (oblimin rotation) accounted for 32.80% of the total variance of the original data (MR1 = 6.35%, MR3 = 6.27%, MR2 = 6.25%, MR4 = 5.50%, MR5 = 5.02%, MR6 = 3.41%).Results of 3 factors
Code
fitted <- fa(indices_wider_bare, 3)
plotly_efa(fitted)Code
parameters::model_parameters(fitted, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR1 | MR2 | MR3 | Complexity | Uniqueness
------------------------------------------------------------------------------
格子卡片.dprime | 0.64 | | | 1.02 | 0.57
美术卡片.dprime | 0.62 | | | 1.02 | 0.57
图片记忆.bps_score | 0.61 | | | 1.22 | 0.66
文字卡片.dprime | 0.56 | | | 1.07 | 0.69
数感.w | 0.47 | | | 1.09 | 0.78
连续再认PRO.dprime | 0.47 | | | 1.17 | 0.79
塔罗牌.nc | 0.45 | | | 1.01 | 0.79
宇宙黑洞A.nc | 0.44 | | | 1.18 | 0.75
捉虫高级简版.dprime | 0.44 | | | 1.65 | 0.76
过目不忘PRO.nc | 0.42 | | | 1.78 | 0.73
登陆月球（中级）.mean_log_err | 0.42 | | | 1.32 | 0.76
幸运小球PRO.nc | 0.39 | | | 1.52 | 0.74
欢乐餐厅PRO.nc | 0.39 | | | 1.06 | 0.84
各得其所.mrt_init | -0.37 | | | 1.98 | 0.83
三维心理旋转测试.nc | 0.37 | | | 2.13 | 0.75
小狗回家.dprime | 0.36 | | | 1.97 | 0.70
顺背数PRO.nc | 0.35 | | | 1.84 | 0.78
数字推理.nc | 0.31 | | | 2.28 | 0.78
方向临摹.mean_log_err | 0.27 | | | 1.48 | 0.90
变戏法.nc | 0.26 | | | 2.88 | 0.74
时间顺序判断.thresh_last_block | 0.24 | | | 2.94 | 0.76
太空飞船PRO.cong_eff_mrt | 0.22 | | | 2.10 | 0.94
时长分辨.thresh_last_block | 0.18 | | | 2.75 | 0.88
小狗回家.mean_score | | 0.64 | | 1.01 | 0.58
一心二用PRO.nc | | 0.64 | | 1.07 | 0.57
连点成画PRO.nc | | 0.55 | | 1.00 | 0.70
火眼金睛.nc | | 0.45 | | 1.31 | 0.72
快速归类PRO.mrt | | 0.41 | | 1.95 | 0.61
萤火虫PRO.nc | | 0.37 | | 1.32 | 0.82
蝴蝶照相机.nc | | 0.34 | | 2.18 | 0.64
位置记忆PRO.nc | | 0.33 | | 2.55 | 0.67
候鸟迁徙PRO.cong_eff_mrt | | 0.25 | | 2.18 | 0.80
超级秒表PRO.mrt | | 0.21 | | 2.91 | 0.81
专注大师_中级.nc | | | 0.76 | 1.01 | 0.44
速算师（中级）.nc | | | 0.71 | 1.02 | 0.44
声调判断.nc | | | 0.58 | 1.01 | 0.67
捉虫高级简版.rtsd | | | 0.31 | 1.75 | 0.84
语义判断.nc | | | 0.29 | 2.68 | 0.72
舒尔特方格（中级）.nc_cor | | | 0.26 | 2.44 | 0.78
多彩文字PRO.cong_eff_mrt | | | 0.20 | 2.26 | 0.89
The 3 latent factors (oblimin rotation) accounted for 27.10% of the total variance of the original data (MR1 = 12.48%, MR2 = 7.51%, MR3 = 7.11%).Code
fitted_sic <- fa(indices_wider_bare_sic, 3)
plotly_efa(fitted_sic)Code
parameters::model_parameters(fitted_sic, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR1 | MR2 | MR3 | Complexity | Uniqueness
-----------------------------------------------------------------------------
图片记忆.bps_score | 0.59 | | | 1.19 | 0.67
格子卡片.dprime | 0.59 | | | 1.01 | 0.65
美术卡片.dprime | 0.54 | | | 1.06 | 0.67
连续再认PRO.dprime | 0.51 | | | 1.05 | 0.74
文字卡片.dprime | 0.51 | | | 1.07 | 0.77
捉虫高级简版.dprime | 0.46 | | | 1.78 | 0.74
宇宙黑洞A.nc | 0.45 | | | 1.15 | 0.74
欢乐餐厅PRO.nc | 0.44 | | | 1.03 | 0.82
塔罗牌.nc | 0.41 | | | 1.01 | 0.82
过目不忘PRO.nc | 0.41 | | | 1.90 | 0.77
小狗回家.dprime | 0.40 | | | 1.95 | 0.66
数感.w | 0.39 | | | 1.07 | 0.86
三维心理旋转测试.nc | 0.37 | | | 2.32 | 0.78
幸运小球PRO.nc | 0.37 | | | 1.47 | 0.80
蝴蝶照相机.nc | 0.33 | | | 2.23 | 0.70
数字推理.nc | 0.30 | | | 2.11 | 0.85
登陆月球（中级）.mean_log_err | 0.29 | | | 1.72 | 0.86
顺背数PRO.nc | 0.29 | | | 1.75 | 0.88
候鸟迁徙PRO.cong_eff_mrt | 0.28 | | | 2.16 | 0.78
方向临摹.mean_log_err | 0.26 | | | 1.27 | 0.92
变戏法.nc | 0.26 | | | 2.71 | 0.77
太空飞船PRO.cong_eff_mrt | 0.24 | | | 1.67 | 0.94
小狗回家.mean_score | | 0.66 | | 1.00 | 0.56
一心二用PRO.nc | | 0.59 | | 1.13 | 0.58
连点成画PRO.nc | | 0.53 | | 1.06 | 0.74
快速归类PRO.mrt | | 0.45 | | 1.96 | 0.57
火眼金睛.nc | | 0.44 | | 1.66 | 0.72
萤火虫PRO.nc | | 0.37 | | 1.28 | 0.83
位置记忆PRO.nc | | 0.35 | | 2.31 | 0.68
各得其所.mrt_init | | 0.30 | | 1.97 | 0.88
舒尔特方格（中级）.nc_cor | | 0.30 | | 1.85 | 0.82
专注大师_中级.nc | | | 0.76 | 1.03 | 0.42
声调判断.nc | | | 0.62 | 1.21 | 0.62
速算师（中级）.nc | | | 0.59 | 1.15 | 0.57
捉虫高级简版.rtsd | | | 0.39 | 1.70 | 0.77
时间顺序判断.thresh_last_block | | | 0.32 | 2.24 | 0.75
语义判断.nc | | | 0.30 | 2.55 | 0.74
超级秒表PRO.mrt | | | 0.24 | 2.33 | 0.83
时长分辨.thresh_last_block | | | 0.24 | 1.82 | 0.89
多彩文字PRO.cong_eff_mrt | | | 0.17 | 1.94 | 0.94
The 3 latent factors (oblimin rotation) accounted for 24.81% of the total variance of the original data (MR1 = 10.98%, MR2 = 7.16%, MR3 = 6.67%).Keep Variables of Low-reliability
In the following we try using all the variables regardless its test-retest reliability.
Code
indices_wider <- indices_pooled |>
filter(!need_trans | (index_type == "trans" & need_trans)) |>
pivot_wider_indices(
name_value = "score",
always_suffix = TRUE
) |>
left_join(
users |>
mutate(school = str_remove(school, "认知实验")) |>
select(user_id, school),
by = "user_id"
)
indices_wider_bare <- select(indices_wider, -user_id, -school)
indices_wider_bare_sic <- indices_wider |>
filter(school == "四川师范大学") |>
select(-user_id, -school)
indices_wider_bare_bj <- indices_wider |>
filter(school != "四川师范大学") |>
select(-user_id, -school)Correlation Matrix
Code
naniar::vis_miss(indices_wider_bare) +
scale_y_continuous(expand = c(0, 0), trans = "reverse") +
theme(axis.text.x = element_text(angle = 90))
Code
vis_cor(indices_wider_bare, quiet = TRUE)
vis_cor(indices_wider_bare_sic, quiet = TRUE)
vis_cor(indices_wider_bare_bj, quiet = TRUE)
Further checking on the correlation matrix of Beijing Data set shows it is not positive definite, either.
Bartlett’s Test
Code
list(
Whole = indices_wider_bare,
`Sichuan Dataset` = indices_wider_bare_sic
) |>
map(
~ psych::cortest.bartlett(.) |>
as_tibble()
) |>
bind_rows(.id = "Data Source") |>
knitr::kable(digits = 2)| Data Source | chisq | p.value | df |
|---|---|---|---|
| Whole | 9335.97 | 0 | 1830 |
| Sichuan Dataset | 6191.12 | 0 | 1830 |
Sampling Adequacy
Code
kmo_result <- psych::KMO(indices_wider_bare)
kmo_result_sic <- psych::KMO(indices_wider_bare_sic)
bind_rows(
whole = kmo_result$MSAi |>
enframe(name = "game_index_name", value = "MSA"),
sichuan = kmo_result_sic$MSAi |>
enframe(name = "game_index_name", value = "MSA"),
.id = "src"
) |>
separate(game_index_name, c("game_name", "index_name"), sep = "\\.") |>
arrange(desc(MSA)) |>
mutate(MSA = digits(MSA, 2)) |>
pivot_wider(
names_from = src,
values_from = MSA,
names_prefix = "MSA_"
) |>
formattable() |>
as.datatable(caption = "KMO Test Result")Code
indices_wider_bare <- select(indices_wider_bare, -候鸟迁徙PRO.switch_cost_mrt)One variable (switch cost for 候鸟迁徙) for the whole data set show too low MSA value (< 0.5). Further analysis does not include it, and further analysis will stick with the whole data set only.
Determining Number of Factors
Code
n_factor_whole <- psych::nfactors(indices_wider_bare)Code
format_n_factors(n_factor_whole)Nine Factor Model
Code
fitted <- fa(indices_wider_bare, 9)
plotly_efa(fitted)Code
parameters::model_parameters(fitted, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR9 | MR2 | MR6 | MR4 | MR3 | MR7 | MR1 | MR5 | MR8 | Complexity | Uniqueness
--------------------------------------------------------------------------------------------------------------------------
格子卡片.dprime | 0.69 | | | | | | | | | 1.06 | 0.40
文字卡片.dprime | 0.65 | | | | | | | | | 1.14 | 0.50
数字卡片PRO.dprime | 0.37 | | | | | | | | | 2.29 | 0.59
察颜观色PRO.switch_cost_mrt | -0.34 | | | | | | | | | 3.72 | 0.80
美术卡片.dprime | 0.32 | | | | | | | | | 3.48 | 0.52
强化学习.pc_test | 0.21 | | | | | | | | | 3.30 | 0.86
小狗回家.mean_score | | 0.61 | | | | | | | | 1.33 | 0.50
一心二用PRO.nc | | 0.53 | | | | | | | | 1.58 | 0.56
连点成画PRO.nc | | 0.50 | | | | | | | | 1.97 | 0.60
快速归类PRO.mrt | | 0.48 | | | | | | | | 2.12 | 0.52
我是大厨.score_total | | 0.44 | | | | | | | | 1.58 | 0.69
捉虫高级简版.mrt | | 0.39 | | | | | | | | 2.55 | 0.67
超级秒表PRO.mrt | | 0.39 | | | | | | | | 2.93 | 0.69
太空飞船PRO.cong_eff_mrt | | -0.28 | | | | | | | | 2.31 | 0.90
言语记忆.fm_dprime | | -0.26 | | | | | | | | 2.98 | 0.89
变戏法.nc | | 0.26 | | | | | | | | 4.23 | 0.69
候鸟迁徙PRO.cong_eff_mrt | | 0.21 | | | | | | | | 3.73 | 0.80
蝴蝶照相机.nc | | | 0.74 | | | | | | | 1.05 | 0.37
位置记忆PRO.nc | | | 0.61 | | | | | | | 1.21 | 0.47
萤火虫PRO.nc | | | 0.37 | | | | | | | 2.30 | 0.73
打靶场.nc | | | 0.37 | | | | | | | 2.26 | 0.58
宇宙黑洞A.nc | | | 0.32 | | | | | | | 2.50 | 0.72
幸运小球PRO.nc | | | | 0.66 | | | | | | 1.30 | 0.43
顺背数PRO.nc | | | | 0.50 | | | | | | 1.34 | 0.63
密码箱.nc | | | | 0.43 | | | | | | 2.23 | 0.56
欢乐餐厅PRO.nc | | | | 0.37 | | | | | | 3.54 | 0.70
过目不忘PRO.nc | | | | 0.32 | | | | | | 3.86 | 0.64
方向临摹.mean_log_err | | | | 0.21 | | | | | | 5.15 | 0.83
速算师（中级）.nc | | | | | 0.70 | | | | | 1.32 | 0.33
专注大师_中级.nc | | | | | 0.69 | | | | | 1.16 | 0.39
声调判断.nc | | | | | 0.40 | | | | | 3.27 | 0.58
多彩文字PRO.cong_eff_mrt | | | | | 0.27 | | | | | 3.26 | 0.81
数字推理.nc | | | | | 0.22 | | | | | 4.76 | 0.75
卡片分类PRO.switch_cost_mrt | | | | | 0.16 | | | | | 5.33 | 0.91
火眼金睛.nc | | | | | | 0.51 | | | | 1.76 | 0.55
雪花收藏家.nc_cor | | | | | | 0.49 | | | | 1.32 | 0.66
图形推理.nc | | | | | | 0.42 | | | | 2.18 | 0.61
舒尔特方格（中级）.nc_cor | | | | | | 0.41 | | | | 3.03 | 0.60
图形折叠.nc | | | | | | 0.36 | | | | 2.21 | 0.77
远距离联想.nc | | | | | | 0.20 | | | | 2.97 | 0.87
三维心理旋转测试.nc | | | | | | | 0.43 | | | 2.56 | 0.59
各得其所.prop_perfect | | | | | | | 0.40 | | | 1.96 | 0.76
各得其所.mrt_init | | | | | | | -0.38 | | | 2.82 | 0.73
人工语言-中级.nc | | | | | | | 0.30 | | | 3.13 | 0.63
按图索骥.mean_log_err_both | | | | | | | 0.30 | | | 4.46 | 0.57
图片记忆.bps_score | | | | | | | 0.29 | | | 4.43 | 0.61
文字推理.nc | | | | | | | 0.25 | | | 5.78 | 0.73
塔罗牌.nc | | | | | | | 0.23 | | | 4.16 | 0.75
登陆月球（中级）.mean_log_err | | | | | | | 0.20 | | | 5.59 | 0.71
捉虫高级简版.rtsd | | | | | | | | 0.60 | | 1.26 | 0.58
捉虫高级简版.dprime | | | | | | | | 0.49 | | 1.55 | 0.65
小狗回家.dprime | | | | | | | | 0.29 | | 4.11 | 0.62
连续再认PRO.dprime | | | | | | | | 0.28 | | 4.04 | 0.74
语义判断.nc | | | | | | | | 0.21 | | 5.73 | 0.65
节奏感知.thresh_last_block | | | | | | | | | 0.57 | 1.36 | 0.61
时长分辨.thresh_last_block | | | | | | | | | 0.42 | 1.95 | 0.70
魔术师终极.dprime | | | | | | | | | 0.34 | 3.20 | 0.64
数感.w | | | | | | | | | 0.32 | 3.46 | 0.69
变色魔块PRO.ssrt | | | | | | | | | 0.32 | 3.32 | 0.77
时间顺序判断.thresh_last_block | | | | | | | | | 0.24 | 5.30 | 0.68
The 9 latent factors (oblimin rotation) accounted for 34.96% of the total variance of the original data (MR9 = 4.51%, MR2 = 4.49%, MR6 = 4.32%, MR4 = 4.07%, MR3 = 3.86%, MR7 = 3.85%, MR1 = 3.56%, MR5 = 3.25%, MR8 = 3.03%).Seven Factor Model
Code
fitted <- fa(indices_wider_bare, 7)
plotly_efa(fitted)Code
parameters::model_parameters(fitted, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR1 | MR7 | MR2 | MR4 | MR6 | MR3 | MR5 | Complexity | Uniqueness
-----------------------------------------------------------------------------------------------------------
格子卡片.dprime | 0.53 | | | | | | | 1.45 | 0.51
文字卡片.dprime | 0.50 | | | | | | | 1.76 | 0.60
各得其所.mrt_init | -0.47 | | | | | | | 1.44 | 0.76
捉虫高级简版.dprime | 0.43 | | | | | | | 1.82 | 0.71
数字卡片PRO.dprime | 0.39 | | | | | | | 2.00 | 0.60
美术卡片.dprime | 0.37 | | | | | | | 2.59 | 0.53
塔罗牌.nc | 0.37 | | | | | | | 1.60 | 0.76
连续再认PRO.dprime | 0.33 | | | | | | | 2.87 | 0.75
图片记忆.bps_score | 0.31 | | | | | | | 3.86 | 0.64
文字推理.nc | 0.31 | | | | | | | 4.11 | 0.75
各得其所.prop_perfect | 0.31 | | | | | | | 2.56 | 0.81
数字推理.nc | 0.30 | | | | | | | 3.16 | 0.75
强化学习.pc_test | 0.22 | | | | | | | 3.00 | 0.86
三维心理旋转测试.nc | | 0.50 | | | | | | 1.95 | 0.60
雪花收藏家.nc_cor | | 0.49 | | | | | | 1.31 | 0.69
火眼金睛.nc | | 0.45 | | | | | | 2.26 | 0.61
图形推理.nc | | 0.42 | | | | | | 2.06 | 0.64
图形折叠.nc | | 0.39 | | | | | | 1.71 | 0.78
按图索骥.mean_log_err_both | | 0.37 | | | | | | 2.51 | 0.63
人工语言-中级.nc | | 0.33 | | | | | | 2.92 | 0.63
登陆月球（中级）.mean_log_err | | 0.24 | | | | | | 3.96 | 0.71
远距离联想.nc | | 0.19 | | | | | | 2.99 | 0.87
小狗回家.mean_score | | | 0.62 | | | | | 1.38 | 0.50
一心二用PRO.nc | | | 0.56 | | | | | 1.33 | 0.57
快速归类PRO.mrt | | | 0.54 | | | | | 1.64 | 0.51
我是大厨.score_total | | | 0.46 | | | | | 1.28 | 0.70
连点成画PRO.nc | | | 0.44 | | | | | 2.27 | 0.64
捉虫高级简版.mrt | | | 0.43 | | | | | 1.92 | 0.68
超级秒表PRO.mrt | | | 0.43 | | | | | 1.90 | 0.70
太空飞船PRO.cong_eff_mrt | | | -0.28 | | | | | 2.64 | 0.89
变戏法.nc | | | 0.27 | | | | | 3.05 | 0.71
言语记忆.fm_dprime | | | -0.26 | | | | | 2.97 | 0.89
候鸟迁徙PRO.cong_eff_mrt | | | 0.23 | | | | | 3.01 | 0.80
幸运小球PRO.nc | | | | 0.71 | | | | 1.04 | 0.45
顺背数PRO.nc | | | | 0.55 | | | | 1.16 | 0.63
密码箱.nc | | | | 0.42 | | | | 2.20 | 0.59
欢乐餐厅PRO.nc | | | | 0.41 | | | | 1.86 | 0.73
过目不忘PRO.nc | | | | 0.32 | | | | 3.57 | 0.69
小狗回家.dprime | | | | 0.24 | | | | 3.86 | 0.68
方向临摹.mean_log_err | | | | 0.22 | | | | 4.21 | 0.85
蝴蝶照相机.nc | | | | | 0.67 | | | 1.13 | 0.42
位置记忆PRO.nc | | | | | 0.60 | | | 1.26 | 0.48
萤火虫PRO.nc | | | | | 0.43 | | | 1.69 | 0.72
打靶场.nc | | | | | 0.31 | | | 2.76 | 0.59
宇宙黑洞A.nc | | | | | 0.28 | | | 2.70 | 0.74
速算师（中级）.nc | | | | | | 0.71 | | 1.28 | 0.33
专注大师_中级.nc | | | | | | 0.67 | | 1.20 | 0.42
声调判断.nc | | | | | | 0.44 | | 2.85 | 0.58
舒尔特方格（中级）.nc_cor | | | | | | 0.31 | | 2.99 | 0.69
多彩文字PRO.cong_eff_mrt | | | | | | 0.24 | | 3.07 | 0.83
察颜观色PRO.switch_cost_mrt | | | | | | 0.16 | | 5.87 | 0.89
变色魔块PRO.ssrt | | | | | | | 0.38 | 1.26 | 0.81
捉虫高级简版.rtsd | | | | | | | 0.33 | 3.95 | 0.69
节奏感知.thresh_last_block | | | | | | | 0.32 | 4.33 | 0.74
时长分辨.thresh_last_block | | | | | | | 0.30 | 2.73 | 0.78
时间顺序判断.thresh_last_block | | | | | | | 0.29 | 3.73 | 0.68
数感.w | | | | | | | 0.29 | 4.36 | 0.70
魔术师终极.dprime | | | | | | | 0.29 | 3.28 | 0.69
语义判断.nc | | | | | | | 0.28 | 3.36 | 0.65
卡片分类PRO.switch_cost_mrt | | | | | | | 0.19 | 3.21 | 0.92
The 7 latent factors (oblimin rotation) accounted for 32.04% of the total variance of the original data (MR1 = 5.60%, MR7 = 5.12%, MR2 = 5.10%, MR4 = 4.79%, MR6 = 4.15%, MR3 = 4.08%, MR5 = 3.20%).Three Factor Model
Code
fitted <- fa(indices_wider_bare, 3)
plotly_efa(fitted)Code
parameters::model_parameters(fitted, sort = TRUE, threshold = "max")# Rotated loadings from Factor Analysis (oblimin-rotation)
Variable | MR1 | MR2 | MR3 | Complexity | Uniqueness
-------------------------------------------------------------------------------
美术卡片.dprime | 0.67 | | | 1.00 | 0.54
格子卡片.dprime | 0.66 | | | 1.01 | 0.57
数字卡片PRO.dprime | 0.58 | | | 1.03 | 0.62
文字卡片.dprime | 0.56 | | | 1.08 | 0.69
图片记忆.bps_score | 0.55 | | | 1.09 | 0.71
人工语言-中级.nc | 0.55 | | | 1.01 | 0.67
数感.w | 0.49 | | | 1.04 | 0.78
按图索骥.mean_log_err_both | 0.49 | | | 1.77 | 0.65
魔术师终极.dprime | 0.49 | | | 1.26 | 0.73
图形推理.nc | 0.48 | | | 1.15 | 0.73
登陆月球（中级）.mean_log_err | 0.47 | | | 1.26 | 0.73
密码箱.nc | 0.44 | | | 1.54 | 0.66
连续再认PRO.dprime | 0.44 | | | 1.04 | 0.82
塔罗牌.nc | 0.44 | | | 1.02 | 0.80
三维心理旋转测试.nc | 0.43 | | | 2.20 | 0.69
文字推理.nc | 0.42 | | | 1.19 | 0.83
过目不忘PRO.nc | 0.42 | | | 1.82 | 0.74
幸运小球PRO.nc | 0.42 | | | 1.36 | 0.73
宇宙黑洞A.nc | 0.41 | | | 1.21 | 0.77
各得其所.mrt_init | -0.40 | | | 1.94 | 0.83
打靶场.nc | 0.39 | | | 2.01 | 0.63
捉虫高级简版.dprime | 0.39 | | | 2.19 | 0.78
顺背数PRO.nc | 0.37 | | | 1.64 | 0.78
欢乐餐厅PRO.nc | 0.37 | | | 1.04 | 0.85
雪花收藏家.nc_cor | 0.36 | | | 1.83 | 0.78
各得其所.prop_perfect | 0.35 | | | 1.13 | 0.88
强化学习.pc_test | 0.34 | | | 1.07 | 0.88
数字推理.nc | 0.34 | | | 1.98 | 0.80
小狗回家.dprime | 0.32 | | | 2.30 | 0.71
方向临摹.mean_log_err | 0.26 | | | 1.26 | 0.91
变戏法.nc | 0.25 | | | 2.90 | 0.73
候鸟迁徙PRO.cong_eff_mrt | 0.25 | | | 2.21 | 0.80
远距离联想.nc | 0.25 | | | 1.45 | 0.89
太空飞船PRO.cong_eff_mrt | 0.24 | | | 1.81 | 0.94
时间顺序判断.thresh_last_block | 0.23 | | | 2.97 | 0.75
时长分辨.thresh_last_block | 0.21 | | | 2.57 | 0.86
变色魔块PRO.ssrt | 0.18 | | | 1.98 | 0.92
小狗回家.mean_score | | 0.62 | | 1.01 | 0.59
一心二用PRO.nc | | 0.59 | | 1.20 | 0.60
连点成画PRO.nc | | 0.57 | | 1.00 | 0.67
火眼金睛.nc | | 0.46 | | 1.39 | 0.69
我是大厨.score_total | | 0.38 | | 1.46 | 0.75
萤火虫PRO.nc | | 0.38 | | 1.46 | 0.80
位置记忆PRO.nc | | 0.34 | | 2.30 | 0.67
蝴蝶照相机.nc | | 0.34 | | 2.21 | 0.66
图形折叠.nc | | 0.34 | | 1.48 | 0.85
节奏感知.thresh_last_block | | 0.24 | | 1.87 | 0.87
舒尔特方格（中级）.nc_cor | | 0.22 | | 2.84 | 0.80
多彩文字PRO.cong_eff_mrt | | 0.18 | | 2.58 | 0.88
言语记忆.fm_dprime | | -0.17 | | 2.52 | 0.96
专注大师_中级.nc | | | 0.71 | 1.01 | 0.48
声调判断.nc | | | 0.57 | 1.03 | 0.67
速算师（中级）.nc | | | 0.52 | 1.28 | 0.59
快速归类PRO.mrt | | | 0.44 | 2.01 | 0.56
捉虫高级简版.rtsd | | | 0.42 | 1.24 | 0.79
捉虫高级简版.mrt | | | 0.37 | 2.07 | 0.72
语义判断.nc | | | 0.29 | 2.55 | 0.71
察颜观色PRO.switch_cost_mrt | | | 0.27 | 2.40 | 0.90
超级秒表PRO.mrt | | | 0.26 | 2.45 | 0.79
卡片分类PRO.switch_cost_mrt | | | 0.23 | 1.58 | 0.95
The 3 latent factors (oblimin rotation) accounted for 24.78% of the total variance of the original data (MR1 = 13.06%, MR2 = 6.30%, MR3 = 5.42%).Additional Analysis
One might be interested which tasks will have the highest loading if we fit only one latent factor. Here is the result (note the variable with low MSA value is still removed):
Code
model_one_fac <- fa(indices_wider_bare, 1)
model_one_fac$loadings |>
unclass() |>
as_tibble(rownames = "variable") |>
mutate(variable = fct_reorder(variable, MR1)) |>
ggplot(aes(variable, MR1)) +
geom_col() +
coord_flip() +
theme_bw() +
labs(y = "Loading") +
theme(axis.title.y = element_blank())
References
Bulmer, M. G. 1979. Principles of Statistics. New York: Dover Publications.
Dziuban, Charles D., and Edwin C. Shirkey. 1974. “When Is a Correlation Matrix Appropriate for Factor Analysis? Some Decision Rules.” Psychological Bulletin 81 (6): 358–61. https://doi.org/10.1037/h0036316.
Howell, David C. 2013. Statistical Methods for Psychology. 8th ed. Belmont, CA: Wadsworth Cengage Learning.
Kline, Rex B. 2016. Principles and Practice of Structural Equation Modeling. Fourth edition. Methodology in the Social Sciences. New York: The Guilford Press.
Petscher, Yaacov M., Christopher Schatschneider, Donald L. Compton, and Yaacov M. Petscher, eds. 2013. Applied Quantitative Analysis in Education and the Social Sciences. New York: Routledge/Taylor & Francis Group.
Woods, Carol M. 2006. “Careless Responding to Reverse-Worded Items: Implications for Confirmatory Factor Analysis.” Journal of Psychopathology and Behavioral Assessment 28 (3): 186–91. https://doi.org/10.1007/s10862-005-9004-7.