From f37c30efe7588a071d9a4f0737c170685778d8fc Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Wed, 19 Feb 2025 16:52:53 +0100
Subject: [PATCH 1/5] =?UTF-8?q?feat:=20script=20R=20pour=20l'export=20de?=
=?UTF-8?q?=20donn=C3=A9es?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
bin/export.r | 144 ++++++++++++++++++
.../sido/gwt/server/ws/QueryServiceImpl.java | 14 +-
2 files changed, 153 insertions(+), 5 deletions(-)
create mode 100755 bin/export.r
diff --git a/bin/export.r b/bin/export.r
new file mode 100755
index 00000000..fd018772
--- /dev/null
+++ b/bin/export.r
@@ -0,0 +1,144 @@
+#!/usr/bin/Rscript --vanilla --no-save --no-restore --quiet --encoding=UTF-8
+# Call SIDO web service to get data
+#
+## Uses
+## - RCurl: https://cran.r-project.org/web/packages/RCurl/index.html
+## - keyring: https://cran.r-project.org/web/packages/keyring/index.html
+## - logger: https://cran.r-project.org/web/packages/logger/index.html
+
+sido_url <- "https://sido.pheno.fr/ws"
+observatory_schema <- "foret"
+
+#' Load or install a package.
+#'
+#' @param packagename string package name
+require_install_package <- function(packagename) {
+ if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+ cat("The library", packagename, "is not installed. Trying to install it.\n")
+ install.packages(packagename, repos = "http://cran.at.r-project.org/")
+ if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+ cat("Installing the library", packagename, "failed.\n")
+ q()
+ }
+ }
+}
+
+require_install_package("RCurl")
+require_install_package("jsonlite")
+require_install_package("keyring")
+require_install_package("logger")
+
+#' Get key value from keyring.
+#'
+#' @param key_name string key name
+#' @param prompt string prompt message
+#' @return string keyring value
+get_keyring_value <- function(key_name, prompt) {
+ logger::log_info("Getting keyring value for {key_name}...")
+ if (!keyring::has_keyring_support()) {
+ logger::log_error("No keyring support!")
+ logger::log_info("Set value for the variable sido_{key_name} in the code.")
+ q()
+ }
+ keyring_service <- "sido-export"
+
+ kr <- keyring::default_backend(keyring = keyring_service)
+ if (!kr$is_available()) {
+ logger::log_error("Keyring {keyring_service} not available!")
+ logger::log_info("Set value for the variable sido_{key_name} in the code.")
+ q()
+ }
+
+ key_list <- kr$list()
+ match_keys <- subset(key_list,
+ service == keyring_service & username == key_name)
+ if (dim(match_keys)[1] == 0) {
+ logger::log_info("No value found for {keyring_service} and {key_name}.")
+ kr$set(service = keyring_service, username = key_name, prompt = prompt)
+ }
+ value <- kr$get(service = keyring_service, username = key_name)
+ value
+}
+
+#' Get access token using client credentials.
+#'
+#' @return access token
+get_access_token <- function(sido_client_id, sido_client_secret) {
+ logger::log_info("Getting access token...")
+ token_url <- paste0(sido_url, "/oauth/token")
+ # Perform the POST request
+ post_data <- paste0(
+ "client_id=", sido_client_id,
+ "&client_secret=", sido_client_secret,
+ "&grant_type=client_credentials")
+ headers <- c("Content-Type" = "application/x-www-form-urlencoded")
+ opts <- list(postfields = post_data, httpheader = headers)
+ logger::log_info("Calling {token_url}...")
+ post_response <- RCurl::postForm(token_url, .opts = opts)
+ logger::log_info("Parsing JSON...")
+ token <- jsonlite::fromJSON(post_response)
+ token$access_token
+}
+
+#' Get queries for a given schema.
+#'
+#' @param schema string schema name
+#' @param access_token string access token
+#' @return queries list of query names
+get_queries <- function(schema, access_token) {
+ logger::log_info("Getting queries...")
+ queries_url <- paste0(sido_url, "/data/queries/", schema)
+ headers <- c("Authorization" = paste0("Bearer ", access_token))
+ queries_response <- RCurl::getURL(queries_url,
+ .opts = list(httpheader = headers))
+ queries <- jsonlite::fromJSON(queries_response)
+ queries$data[, "name"]
+}
+
+#' Get data for a given schema and query.
+#'
+#' @param schema string schema name
+#' @param query string query name
+#' @param access_token string access token
+#' @return data data frame
+get_data <- function(schema, query, access_token) {
+ logger::log_info("- Getting data for {schema}/{query}...")
+ data_url <- paste0(sido_url, "/data/", schema, "/", query)
+ headers <- c("Authorization" = paste0("Bearer ", access_token))
+ data_response <- RCurl::getURL(data_url,
+ .opts = list(httpheader = headers),
+ .encoding = "UTF-8")
+ data <- jsonlite::fromJSON(data_response)
+ if (data$status != 200) {
+ logger::log_error("- Error ", data$status, ": ", data$message, "\n")
+ }
+ df <- data.frame(data$data$values)
+ names(df) <- data$data$properties$title
+ df
+}
+
+sido_client_id <- get_keyring_value("client_id",
+ "Your client_id for SIDO")
+sido_client_secret <- get_keyring_value("client_secret",
+ "Your client_secret for SIDO")
+
+token <- get_access_token(sido_client_id, sido_client_secret)
+queries <- get_queries(observatory_schema, token)
+logger::log_info("- nb of queries: {0}", length(queries))
+for (query in queries) {
+ logger::log_info("- query: {query}")
+ if (query == "data") {
+ logger::log_info(" - skipping data query")
+ } else {
+ data <- get_data(observatory_schema, query, token)
+ filename <- paste0(observatory_schema, "-", query, ".csv")
+ logger::log_info(" - file: {filename}...")
+ write.table(data,
+ file = filename,
+ col.names = TRUE,
+ row.names = FALSE,
+ sep = ",",
+ quote = TRUE)
+ logger::log_info(" done")
+ }
+}
diff --git a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
index 73619af3..8a5a10b8 100644
--- a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
+++ b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
@@ -112,12 +112,16 @@ public class QueryServiceImpl implements QueryService {
@Override
public final WsQuery getQuery(@NonNull final Datasource obs,
- final String queryName) {
- if (getQueries(obs) != null) {
- return getQueries(obs).get(0);
- } else {
- return null;
+ @NonNull final String queryName) {
+ final var queries = getQueries(obs);
+ if (queries != null) {
+ for (final var query : queries) {
+ if (queryName.equals(query.getName())) {
+ return query;
+ }
+ }
}
+ return null;
}
@Override
--
GitLab
From c8c93907038a5fcc24e3f140dfea56b26983f673 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Wed, 19 Feb 2025 17:19:17 +0100
Subject: [PATCH 2/5] Revue de code
---
bin/export.r | 4 ++--
.../soeretempo/sido/gwt/server/ws/QueryServiceImpl.java | 9 ++++-----
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/bin/export.r b/bin/export.r
index fd018772..a8c33dd2 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -14,10 +14,10 @@ observatory_schema <- "foret"
#' @param packagename string package name
require_install_package <- function(packagename) {
if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
- cat("The library", packagename, "is not installed. Trying to install it.\n")
+ logger::log_info("Installing the library {packagename}...")
install.packages(packagename, repos = "http://cran.at.r-project.org/")
if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
- cat("Installing the library", packagename, "failed.\n")
+ logger::log_error("Installing the library {packagename} failed.")
q()
}
}
diff --git a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
index 8a5a10b8..b18d308c 100644
--- a/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
+++ b/sido-gwt/src/main/java/fr/soeretempo/sido/gwt/server/ws/QueryServiceImpl.java
@@ -115,11 +115,10 @@ public class QueryServiceImpl implements QueryService {
@NonNull final String queryName) {
final var queries = getQueries(obs);
if (queries != null) {
- for (final var query : queries) {
- if (queryName.equals(query.getName())) {
- return query;
- }
- }
+ return queries.stream()
+ .filter(query -> queryName.equals(query.getName()))
+ .findFirst()
+ .orElse(null);
}
return null;
}
--
GitLab
From e3de4a49a442c6f2e76f27aada799f833dfa0c64 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Wed, 19 Feb 2025 17:22:14 +0100
Subject: [PATCH 3/5] Revue de code
---
bin/export.r | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/export.r b/bin/export.r
index a8c33dd2..ecfcddeb 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -110,7 +110,7 @@ get_data <- function(schema, query, access_token) {
.encoding = "UTF-8")
data <- jsonlite::fromJSON(data_response)
if (data$status != 200) {
- logger::log_error("- Error ", data$status, ": ", data$message, "\n")
+ logger::log_error(paste0(" Error ", data$status, ": ", data$message))
}
df <- data.frame(data$data$values)
names(df) <- data$data$properties$title
--
GitLab
From a94b4ada9ec9841c0c6b57e0117515d77c7bd9d5 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Thu, 20 Feb 2025 16:43:37 +0100
Subject: [PATCH 4/5] refactor: prise en compte de la revue de code R de
Patrice Lecharpentier
---
bin/export.r | 67 ++++++++++++++++++++++++++++++++--------------------
1 file changed, 42 insertions(+), 25 deletions(-)
diff --git a/bin/export.r b/bin/export.r
index ecfcddeb..8a225f1e 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -6,21 +6,23 @@
## - keyring: https://cran.r-project.org/web/packages/keyring/index.html
## - logger: https://cran.r-project.org/web/packages/logger/index.html
-sido_url <- "https://sido.pheno.fr/ws"
+sido_ws_url <- "https://sido.pheno.fr/ws"
observatory_schema <- "foret"
+keyring_service <- "sido-export"
#' Load or install a package.
#'
#' @param packagename string package name
require_install_package <- function(packagename) {
- if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+ if (!library(packagename, character.only=TRUE, logical.return=TRUE)) {
logger::log_info("Installing the library {packagename}...")
install.packages(packagename, repos = "http://cran.at.r-project.org/")
- if (FALSE == require(packagename, quietly = TRUE, character.only = TRUE)) {
+ if (!library(packagename, character.only=TRUE, logical.return=TRUE)) {
logger::log_error("Installing the library {packagename} failed.")
q()
}
}
+ require(packagename, quietly = TRUE, character.only = TRUE)
}
require_install_package("RCurl")
@@ -28,26 +30,33 @@ require_install_package("jsonlite")
require_install_package("keyring")
require_install_package("logger")
+#' Get keyring
+#'
+#' @return keyring for the script
+get_keyring <- function(keyring_service) {
+ logger::log_info("Using backend_file")
+ #kr <- keyring::backend_file$new(keyring = keyring_service)
+ kr <- keyring::default_backend(keyring = keyring_service)
+ if (kr$keyring_is_locked(keyring=keyring_service)) {
+ kr$keyring_unlock(keyring = keyring_service)
+ }
+ kr
+}
+
#' Get key value from keyring.
#'
+#' @param kr keyring for the script
#' @param key_name string key name
#' @param prompt string prompt message
#' @return string keyring value
-get_keyring_value <- function(key_name, prompt) {
+get_keyring_value <- function(kr, key_name, prompt) {
logger::log_info("Getting keyring value for {key_name}...")
if (!keyring::has_keyring_support()) {
logger::log_error("No keyring support!")
logger::log_info("Set value for the variable sido_{key_name} in the code.")
q()
}
- keyring_service <- "sido-export"
-
- kr <- keyring::default_backend(keyring = keyring_service)
- if (!kr$is_available()) {
- logger::log_error("Keyring {keyring_service} not available!")
- logger::log_info("Set value for the variable sido_{key_name} in the code.")
- q()
- }
+ keyring_service <- kr$keyring_default()
key_list <- kr$list()
match_keys <- subset(key_list,
@@ -56,14 +65,16 @@ get_keyring_value <- function(key_name, prompt) {
logger::log_info("No value found for {keyring_service} and {key_name}.")
kr$set(service = keyring_service, username = key_name, prompt = prompt)
}
- value <- kr$get(service = keyring_service, username = key_name)
- value
+ kr$get(service = keyring_service, username = key_name)
}
#' Get access token using client credentials.
#'
+#' @param sido_url string root URL of SIDO WS
+#' @param sido_client_id string OAuth2 client ID for SIDO
+#' @param sido_client_secret string OAuth2 client secret for SIDO
#' @return access token
-get_access_token <- function(sido_client_id, sido_client_secret) {
+get_access_token <- function(sido_url, sido_client_id, sido_client_secret) {
logger::log_info("Getting access token...")
token_url <- paste0(sido_url, "/oauth/token")
# Perform the POST request
@@ -82,10 +93,11 @@ get_access_token <- function(sido_client_id, sido_client_secret) {
#' Get queries for a given schema.
#'
+#' @param sido_url string root URL of SIDO WS
#' @param schema string schema name
#' @param access_token string access token
#' @return queries list of query names
-get_queries <- function(schema, access_token) {
+get_queries <- function(sido_url, schema, access_token) {
logger::log_info("Getting queries...")
queries_url <- paste0(sido_url, "/data/queries/", schema)
headers <- c("Authorization" = paste0("Bearer ", access_token))
@@ -97,11 +109,12 @@ get_queries <- function(schema, access_token) {
#' Get data for a given schema and query.
#'
+#' @param sido_url string root URL of SIDO WS
#' @param schema string schema name
#' @param query string query name
#' @param access_token string access token
#' @return data data frame
-get_data <- function(schema, query, access_token) {
+get_data <- function(sido_url, schema, query, access_token) {
logger::log_info("- Getting data for {schema}/{query}...")
data_url <- paste0(sido_url, "/data/", schema, "/", query)
headers <- c("Authorization" = paste0("Bearer ", access_token))
@@ -117,20 +130,24 @@ get_data <- function(schema, query, access_token) {
df
}
-sido_client_id <- get_keyring_value("client_id",
- "Your client_id for SIDO")
-sido_client_secret <- get_keyring_value("client_secret",
- "Your client_secret for SIDO")
+# Run
+
+kr <- get_keyring(keyring_service)
+sido_client_id <- get_keyring_value(kr, "client_id",
+ "Type your client_id for SIDO")
+sido_client_secret <- get_keyring_value(kr, "client_secret",
+ "Type your client_secret for SIDO")
-token <- get_access_token(sido_client_id, sido_client_secret)
-queries <- get_queries(observatory_schema, token)
+token <- get_access_token(sido_ws_url, sido_client_id, sido_client_secret)
+queries <- get_queries(sido_ws_url, observatory_schema, token)
logger::log_info("- nb of queries: {0}", length(queries))
-for (query in queries) {
+for (i in seq_along(queries)) {
+ query = queries[i]
logger::log_info("- query: {query}")
if (query == "data") {
logger::log_info(" - skipping data query")
} else {
- data <- get_data(observatory_schema, query, token)
+ data <- get_data(sido_ws_url, observatory_schema, query, token)
filename <- paste0(observatory_schema, "-", query, ".csv")
logger::log_info(" - file: {filename}...")
write.table(data,
--
GitLab
From ef1910909fe50b79b454f63017e3aab35d548664 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Fri, 21 Feb 2025 10:21:04 +0100
Subject: [PATCH 5/5] Suppression de commentaire
---
bin/export.r | 1 -
1 file changed, 1 deletion(-)
diff --git a/bin/export.r b/bin/export.r
index 8a225f1e..1fdc4f3a 100755
--- a/bin/export.r
+++ b/bin/export.r
@@ -35,7 +35,6 @@ require_install_package("logger")
#' @return keyring for the script
get_keyring <- function(keyring_service) {
logger::log_info("Using backend_file")
- #kr <- keyring::backend_file$new(keyring = keyring_service)
kr <- keyring::default_backend(keyring = keyring_service)
if (kr$keyring_is_locked(keyring=keyring_service)) {
kr$keyring_unlock(keyring = keyring_service)
--
GitLab