This program is not for hw1. The link for hw1 is in README.md.
Our project is aimed to analyze the relation between usage of English words and number of examples on several famous online dictionaries. In other words, we want to see how much example sentences we can find when we look up a English word. The wordlist will be prepared by 何國瑋, and the program for which can be found here. I wrote three similar programs to count the number of sentence examples on www.dictionary.com, www.lexico.com (by Oxford) and https://dictionary.cambridge.org/. As mentioned, a wordlist like this is needed as an input file:
input = read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/sampletext.csv", header = FALSE)
input## V1 V2
## 1 test 5
## 2 math 8
## 3 steak 23
## 4 bald 22
## 5 girl 21
## 6 friend 33
## 7 the 200
## 8 avocado 2
## 9 mouse 8
## 10 book 50The first column contains the word, while the second column contains the frequency of occurence. Now, here is the crawlers that look for the examples:
dictionary.com
library(dplyr)##
## Attaching package: 'dplyr'## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, unionlibrary(rvest)## Loading required package: xml2#load input wordlist
word = read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/sampletext.csv", header = TRUE)
word$exNum = 0
time = c(Sys.time(), Sys.time(), Sys.time(), Sys.time(), Sys.time(), Sys.time() )
for(i in 1:NROW(word))
{
t_i = Sys.time()
dictionaryPage =
tryCatch(read_html(paste("https://www.dictionary.com/browse/", word[i, 1], sep = "")),
error = function(e){
word[i, 3] <<- -1
}
)
if(word[i, 3] != -1)
{
exampleUnderDefinitions = html_nodes(dictionaryPage, ".luna-example")
exampleInExampleSection = html_nodes(dictionaryPage, ".e15kc6du7")
exampleCount = NROW(exampleUnderDefinitions) + NROW(exampleInExampleSection)
word[i, 3] = exampleCount
}
if (i < 7)
{
time[i] = Sys.time()
}
else
{
for(k in 1:5)
{
time[k] = time [k+1]
}
time[6] = Sys.time()
}
if( i > 5 )
{
cat(i, "/", NROW(word), " ", as.character(word[i, 1]), " freq: ", word[i, 2], " ex = ", word[i, 3], " ETA:", (time[6] - time[1]) / 5 * ( NROW(word) - i ) , "s \n")
}
else{
cat(i, "/", NROW(word), " ", as.character(word[i, 1]), " freq: ", word[i, 2], " ex = ", word[i, 3], "\n")
}
}## 1 / 9 math freq: 8 ex = 10
## 2 / 9 steak freq: 23 ex = 10
## 3 / 9 bald freq: 22 ex = 15
## 4 / 9 girl freq: 21 ex = 13
## 5 / 9 friend freq: 33 ex = 14
## 6 / 9 the freq: 200 ex = 18 ETA: 4.316722 s
## 7 / 9 avocado freq: 2 ex = 10 ETA: 2.994011 s
## 8 / 9 mouse freq: 8 ex = 11 ETA: 1.892224 s
## 9 / 9 book freq: 50 ex = 36 ETA: 0 swrite.csv(word, file = "~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/dictionary_com_output_sample.csv")Results:
read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/dictionary_com_output_sample.csv")## X test X5 exNum
## 1 1 math 8 10
## 2 2 steak 23 10
## 3 3 bald 22 15
## 4 4 girl 21 13
## 5 5 friend 33 14
## 6 6 the 200 18
## 7 7 avocado 2 10
## 8 8 mouse 8 11
## 9 9 book 50 36www.lexico.com
library(dplyr)
library(rvest)
#load input wordlist
word = read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/sampletext.csv", header = TRUE)
word$exNum = 0
time = c(Sys.time(), Sys.time(), Sys.time(), Sys.time(), Sys.time(), Sys.time() )
for(i in 1:NROW(word))
{
dictionaryPage =
tryCatch(read_html(paste("https://www.lexico.com/en/definition/", word[i, 1], sep = "")),
error = function(e){
word[i, 3] <<- -1
}
)
if(word[i, 3] != -1)
{
exampleCount = NROW(html_nodes(dictionaryPage, "em"))
word[i, 3] = exampleCount
}
if (i < 7)
{
time[i] = Sys.time()
}
else
{
for(k in 1:5)
{
time[k] = time [k+1]
}
time[6] = Sys.time()
}
if( i > 5 )
{
cat(i, "/", NROW(word), " ", as.character(word[i, 1]), " freq: ", word[i, 2], " ex = ", word[i, 3], " ETA:", (time[6] - time[1]) / 5 * ( NROW(word) - i ) , "s \n")
}
else{
cat(i, "/", NROW(word), " ", as.character(word[i, 1]), " freq: ", word[i, 2], " ex = ", word[i, 3], "\n")
}
}## 1 / 9 math freq: 8 ex = 34
## 2 / 9 steak freq: 23 ex = 67
## 3 / 9 bald freq: 22 ex = 105
## 4 / 9 girl freq: 21 ex = 71
## 5 / 9 friend freq: 33 ex = 194
## 6 / 9 the freq: 200 ex = 87 ETA: 7.420277 s
## 7 / 9 avocado freq: 2 ex = 59 ETA: 4.837598 s
## 8 / 9 mouse freq: 8 ex = 87 ETA: 2.535104 s
## 9 / 9 book freq: 50 ex = 422 ETA: 0 swrite.csv(word, file = "~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/oxford_output_sample.csv")Results:
read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/oxford_output_sample.csv")## X test X5 exNum
## 1 1 math 8 34
## 2 2 steak 23 67
## 3 3 bald 22 105
## 4 4 girl 21 71
## 5 5 friend 33 194
## 6 6 the 200 87
## 7 7 avocado 2 59
## 8 8 mouse 8 87
## 9 9 book 50 422dictionary.cambridge.org
library(dplyr)
library(rvest)
#load input wordlist
word = read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/sampletext.csv", header = TRUE)
word$exNum = 0
time = c(Sys.time(), Sys.time(), Sys.time(), Sys.time(), Sys.time(), Sys.time() )
for(i in 1:NROW(word))
{
dictionaryPage =
tryCatch(read_html(paste("https://dictionary.cambridge.org/dictionary/english/", word[i, 1], sep = "")),
error = function(e){
word[i, 3] <<- -1
}
)
if(word[i, 3] != -1)
{
exampleCount = NROW(html_nodes(dictionaryPage, ".eg"))
word[i, 3] = exampleCount
}
if (i < 7)
{
time[i] = Sys.time()
}
else
{
for(k in 1:5)
{
time[k] = time [k+1]
}
time[6] = Sys.time()
}
if( i > 5 )
{
cat(i, "/", NROW(word), " ", as.character(word[i, 1]), " freq: ", word[i, 2], " ex = ", word[i, 3], " ETA:", (time[6] - time[1]) / 5 * ( NROW(word) - i ) , "s \n")
}
else{
cat(i, "/", NROW(word), " ", as.character(word[i, 1]), " freq: ", word[i, 2], " ex = ", word[i, 3], "\n")
}
}## 1 / 9 math freq: 8 ex = 0
## 2 / 9 steak freq: 23 ex = 18
## 3 / 9 bald freq: 22 ex = 27
## 4 / 9 girl freq: 21 ex = 32
## 5 / 9 friend freq: 33 ex = 39
## 6 / 9 the freq: 200 ex = 68 ETA: 8.514556 s
## 7 / 9 avocado freq: 2 ex = 17 ETA: 5.043042 s
## 8 / 9 mouse freq: 8 ex = 13 ETA: 1.519365 s
## 9 / 9 book freq: 50 ex = 70 ETA: 0 swrite.csv(word, file = "~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/cambridge_output_sample.csv")Results:
read.csv("~/Documents/summerproj/data_science_programming/week_1/day2/dictionary/cambridge_output_sample.csv")## X test X5 exNum
## 1 1 math 8 0
## 2 2 steak 23 18
## 3 3 bald 22 27
## 4 4 girl 21 32
## 5 5 friend 33 39
## 6 6 the 200 68
## 7 7 avocado 2 17
## 8 8 mouse 8 13
## 9 9 book 50 70