Crawl and extract data
Grabbing data from APIs
Export and read Data
data wrangling & analysis
Grab keywords search volume from DataForSeo API using R'
What is DataForSeo?
DataForSEO is an all-in-one paid API that provides SEO data.
it allows, for example, to retrieve keywords ranking and keyword search volume directly from Google Ads or Bings Ads.
How to use the DataForSeo API?
When you create your account, you are given a few dollars to test the service.
API Authentication
⚠️ Developer credentials (such as passwords, keys and client IDs) should be kept confidential. ⚠️
1
# We need to load a few packages to run this script
2
#
3
# If you don't have them installed yet,
4
# you need to run the dedicated instruction: install.packages('demoPackageName')
5
6
# Package for working with HTTP requests
7
library(httr)
8
9
# Package for working with JSON files
10
library(jsonlite)
11
12
13
# here you will the values with your own credentials
14
# They can be found here https://app.dataforseo.com/api-dashboard
15
16
username <- "APILOGIN"
17
password <- "APIPWD"
18
19
# This will create a header that we'll use to authenticate ourselves each time we use the API.
20
# This code block needs to be kept private as anyone could use your credits.
21
22
headers = c(
23
`Authorization` = paste('Basic',base64_enc(paste0(username,":",password))),
24
`Content-Type` = 'application/json'
25
)
Copied!
Request Google Ads Search Volume for a keyword
Before making the script run over a list, we'll run it for the keyword and break down each steps
1
# here is the list of the parameters of our request
2
data = paste0('[{"device":"all", "search_partners":false, "keywords":["','R for SEO',
3
'"], "location_code":2840, "language_code":"en", "sort_by":"search_volume"}]')
4
5
6
# keyword: R for SEO
7
# device: all, we want desktop, tablet and mobile search volume
8
# search_partners:false, because If we are doing SEO, we don't care about Google Ads being displayed outside of Google Search.
9
# location_code":2840 is for the united states
10
# all the location codes can be downloaded from this page https://docs.dataforseo.com/v3/serp/google/locations/?bash
11
# language_code:en, we want English
12
# sort_by: search_volume, this only matters when requesting several keywords at the same time.
13
14
# This is our data request
15
res <- httr::POST(url = 'https://api.dataforseo.com/v3/keywords_data/google_ads/search_volume/live', httr::add_headers(.headers=headers), body = data)
16
# the httr:: is here to be sure to use the function inside the httr package
17
18
# This will transcribe the reply into characters
19
res_text <- httr::content(res, "text")
20
21
# This will transcribe between JSON format data to an easy to read R object.
22
res_json <- jsonlite::fromJSON(res_text, flatten = TRUE)
23
Copied!
Search volume can be displayed by using this cmd in the terminal.
1
res_json[["tasks"]][["result"]][[1]][["search_volume"]]
Copied!
If the reply is
NULL something is wrong and you might want to explore the full API response by displaying it in full like this:1
View(res_json)
Copied!
this is the response from an API request that worked as shown by the success = TRUE at the top
Click on the blue arrows to view details
If you are happy with the results you can now save the value.
1
search_volume <- res_json[["tasks"]][["result"]][[1]][["search_volume"]]
Copied!
I would suggest to also store the 'competition', 'competition_index', 'low_top_of_page_bid', 'high_top_of_page_bid'.
Better to have more data, its up to you to use it or not later.
1
competition <- res_json[["tasks"]][["result"]][[1]][["competition"]]
2
competition_index <- res_json[["tasks"]][["result"]][[1]][["competition_index"]]
3
klow_top_of_page_bid <- res_json[["tasks"]][["result"]][[1]][["low_top_of_page_bid"]]
4
high_top_of_page_bid <- res_json[["tasks"]][["result"]][[1]][["high_top_of_page_bid"]]
Copied!
Now that the script is validated we can run the script run over the full list
Request Google Ads Search Volume for a batch of keywords
The first step is to load our keywords list
1
# This will prompt a file selector which can be a text file or a CSV file,
2
# as long as, if its a csv, there is keywords in the first one column
3
kwds <- read.csv(file.choose())
4
5
# This will remove duplicate values
6
kwds <- unique(kwds)
7
8
# We will rename the first column name for convenience
9
# and to make the rest of the R script easier to read
10
colnames(kwds)[1] <- "Kwd"
Copied!
Then we run a keyword request through a loop
1
for(i in 1:nrow(kwds)) { # for-loop over rows
2
3
# if the search volume is already defined we skip to next keyword
4
if(is.null(kwds[i,"search_volume"]) || is.na(kwds[i,"search_volume"])){
5
6
7
data = paste0('[{"device":"all", "search_partners":false, "keywords":["',kwds[i,"Kwd"],
8
'"], "location_code":2840, "language_code":"en", "sort_by":"search_volume"}]')
9
10
# We don't want the script to stop if one query fails.
11
# So we are using a tryCatch function to avoid that
12
tryCatch(
13
expr = {
14
15
16
res <- httr::POST(url = 'https://api.dataforseo.com/v3/keywords_data/google_ads/search_volume/live', httr::add_headers(.headers=headers), body = data)
17
res_text <- httr::content(res, "text")
18
res_json <- jsonlite::fromJSON(res_text, flatten = TRUE)
19
20
kwds[i,"search_volume"] <- res_json[["tasks"]][["result"]][[1]][["search_volume"]]
21
kwds[i,"competition"] <- res_json[["tasks"]][["result"]][[1]][["competition"]]
22
kwds[i,"competition_index"] <- res_json[["tasks"]][["result"]][[1]][["competition_index"]]
23
kwds[i,"low_top_of_page_bid"] <- res_json[["tasks"]][["result"]][[1]][["low_top_of_page_bid"]]
24
kwds[i,"high_top_of_page_bid"] <- res_json[["tasks"]][["result"]][[1]][["high_top_of_page_bid"]]
25
26
message(i, " ",kwds[i,"Kwd"], " ok")
27
28
# (Optional)
29
# make the script sleep between each request
30
# we don't want to go over the API hit rate limit
31
Sys.sleep(2)
32
33
# (Optional)
34
# save on the hard drive the results, for the paranoid
35
write.csv(kwds, "kwds.csv")
36
},
37
error = function(e){
38
# (Optional)
39
# Do this if an error is caught...
40
message(i, " ",kwds[i,"Kwd"], " error")
41
},
42
warning = function(w){
43
# (Optional)
44
# Do this if an warning is caught...
45
message(i, " ",kwds[i,"Kwd"], " warning")
46
},
47
finally = {
48
# (Optional)
49
# Do this at the end before quitting the tryCatch structure...
50
}
51
)
52
53
54
}
55
}
Copied!
Request Google Ads Search Volume for a big batch of keywords
⚠️ DataForSEO is actually charging per request. So if you have lots of keywords to check will be much cheaper to group keywords.
This is the script that will help you request queries in batches of 100.
1
pas <- 100
2
i <- 1
3
4
# we prepare the kwds to receive the data by adding the proper column
5
kwds[ ,c("spell", "location_code", "language_code", "search_partners", "competition", "competition_index","search_volume", "low_top_of_page_bid", "high_top_of_page_bid")] <- NA
6
7
for(i in seq(from=1, to=nrow(kwds), by=pas)){ # for-loop over rows
8
9
if(is.null(kwds[i,"search_volume"]) || is.na(kwds[i,"search_volume"])){
10
11
# building the list of kwd to request
12
data <- paste0('[{"device":"all", "search_partners":false, "keywords":["', kwds[i,"Kwd"])
13
for (idkwd in 1:(pas-1)) {
14
if(!is.null(kwds[i+idkwd,"Kwd"]) && !is.na(kwds[i+idkwd,"Kwd"])){
15
data <- paste0(data,'", "',kwds[i+idkwd,"Kwd"])
16
}
17
}
18
data <- paste0(data, '"], "location_code":2840, "language_code":"en", "sort_by":"search_volume"}]')
19
20
21
22
tryCatch(
23
expr = {
24
25
26
res <- httr::POST(url = 'https://api.dataforseo.com/v3/keywords_data/google_ads/search_volume/live', httr::add_headers(.headers=headers), body = data)
27
res_text <- httr::content(res, "text")
28
res_json <- jsonlite::fromJSON(res_text, flatten = TRUE)
29
30
# cleaning results
31
batch <- as.data.frame(do.call(cbind, res_json[["tasks"]][["result"]][[1]]))
32
batch <- data.frame(lapply(batch, as.character), stringsAsFactors=FALSE)
33
data.table::setnames(batch, "keyword", "Kwd")
34
batch$monthly_searches <- NULL
35
36
# inserting result inside our main data frame kwds
37
kwds[match(batch$Kwd, kwds$Kwd), ] <- batch
38
39
message(i, " ",kwds[i,"Kwd"], " OK")
40
41
# (Optional)
42
# make the script sleep between each request
43
# we don't want to go over the API hit rate limit
44
Sys.sleep(5)
45
46
47
# (Optional)
48
# save on the hard drive the results, for the paranoid
49
write.csv(kwds, "kwds.csv")
50
},
51
error = function(e){
52
# (Optional)
53
# Do this if an error is caught...
54
message(i, " ",kwds[i,"Kwd"], " error")
55
break
56
},
57
warning = function(w){
58
# (Optional)
59
# Do this if an warning is caught...
60
message(i, " ",kwds[i,"Kwd"], " warning")
61
},
62
finally = {
63
# (Optional)
64
# Do this at the end before quitting the tryCatch structure...
65
}
66
)
67
68
69
}
70
}
Copied!