##  Aim: validate the EN_100k vector space against a set of behavioral data  ####
## We are going to use reaction times from the British Lexicon Project
## We use again the LSAfun library and load the EN_100k vector space
library(LSAfun)

# use either of the two follwoing lines, replacing "~/dataTeaching/" with the path to the folder with your data
load("~/dataTeaching/EN_100k.rda")
load(file.choose())


# load a second dataset for validation ####

# again, either line is fine
load("~/dataTeaching/blp-items.Rdata")
load(file.choose())

# do some basic checking: head(), tail(), str(), dim()
head(blp.items)
dim(blp.items)

# Select words ending on a specific affix ####

# check whether we can again use words ending in -ity
blp.items[grep(blp.items$spelling,pattern="*ity$"),"spelling"]
# [1] bity   guity  city   fruity pity  

# No, there are just these five, and they mostly do not look like complex words
# We switch to a different suffix: -ness

# display all words (="spelling") ending in -ness
blp.items[grepl(blp.items$spelling,pattern="*ness$"),"spelling"]

# form a subset
blpSelection <- blp.items[grepl(blp.items$spelling,pattern="*ness$"),]
dim(blpSelection)
head(blpSelection)

# add a column to the dataframe containing the bases: we strip "ness" from the end of the words

# recall the sub() syntax:
sub(blpSelection$spelling,pattern="ness$",replacement="")

# create a new column in the blpSelection dataframe
blpSelection$base <- sub(blpSelection$spelling,pattern="ness$",replacement="")
head(blpSelection)

# check against items in vectorspace ####
# now we check whether the bases and derivations are also in our vectorspace

# basic approach: check whether each item also occurs as a row name in the vector space
is.element(blpSelection$base,row.names(EN_100k))
is.element(blpSelection$spelling,row.names(EN_100k))

# in more detail: we create smaller and smaller subsets
# important: remember to use the smaller subsets once you have created it

# first, exlude rows for which the base does not occur in the vectorspace
smallDataset <- blpSelection[is.element(blpSelection$base,row.names(EN_100k)),]
dim(smallDataset)
dim(blpSelection)

# second: same procedure for the -ness forms
smallerDataset <- smallDataset[is.element(smallDataset$spelling,row.names(EN_100k)),]

dim(smallerDataset)
# [1] 109   9
head(smallerDataset)


# Calculate cosine similarites and check correlations  ####

# Calculate similarities between base and derivation and save them in a new column
# NB: since the pairwise() command comes from the library "LSAfun", you need to have run library(LSAfun) earlier,
smallerDataset$cosSim <- pairwise(smallerDataset$base,smallerDataset$spelling,tvectors=EN_100k)

head(smallerDataset)

# run a correlation test
cor.test(smallerDataset$rt,smallerDataset$cosSim)

# The result is somewhat disappointing: a small effect, and barely significant (very close to 0.05)
# But if you think about it, there are many additional factors we did not consider. For example, length and frequency probably influence reaction times
# Can we do better? One easy, still very crude, fix is to use the offsets of the RTs, that is, the result of subtracting the RT of the base from the RT of the -ness form.
# This might at least get rid of the length effect.


## We stopped here in class :)
##
## But, if you are curious ... :)


# Explore what happens if we include the reaction times for the bases 

# pull out all the bases and store them in a vector 
bases <- smallerDataset$base
bases

# create a new dataframe containing just the bases and their reaction times from the BLP
baseAndRts <- blp.items[is.element(blp.items$spelling,bases),c("spelling","rt")]
baseAndRts

# rename the two columns (in order to avoid two columns with the same name in the final dataframe)
names(baseAndRts) <- c("base","baseRT")

head(baseAndRts)

# combine with our old dataframe
# Note: merge() is a very complex, powerful command. In our case, it works as if by magic: in combining the two dataframes, only those rows are kept which have the same "base" entry in both single dataframes
# that is, we combine and subset the larger frame at the same time.
dim(merge(baseAndRts,smallerDataset))

finalSet <- merge(baseAndRts,smallerDataset)
head(finalSet)

# create the RT offsets by subtracting the base RTs from the derivative RTs
finalSet$offset <- finalSet$rt -finalSet$baseRT

# check the correlation again
cor.test(finalSet$offset,finalSet$cosSim)

# Stronger effect, and much more reliable (= lower p-value)
# We will talk more about correlation in later sessions :)