data import from ownCloud/LRB - DataProcess implemented, now all data…

… (WQ, flow, weather) can be used
joboog · Jan 25, 2018 · 3cd9a24 · 3cd9a24
1 parent 793b91e
commit 3cd9a24
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 21 deletions.
diff --git a/global.R b/global.R
@@ -21,6 +21,9 @@ library(tidyr)
 
 wd <- getwd()
 
+data_dir <- "E:/ownCloud/LRB - DataProcess"
+
+
 # support scripts ===
 
 # load source script for summary table ===
@@ -36,10 +39,108 @@ countValues <- function(x){
 
 # set data dir
 if (dir.exists("Data_In")==TRUE) {
-  data_dir <- "/Data_In/"
+  data_dir <- "E:/ownCloud/LRB - DataProcess"
+
+  #=== load data from ownCloud ====================================
+  load(file = paste0(data_dir, "/03 R_formats/LRB_WQ.rda"))
+  load(file = paste0(data_dir, "/03 R_formats/LRB_Weather.rda"))
+  load(file = paste0(data_dir, "/03 R_formats/LRB_Flow.rda"))
+
+
+
+
+  # load samplepoint info
+  SPoint.dat <- read.csv(paste0(data_dir,"/02 Management_CSV/LRB_SamplePoint.csv"), header=TRUE, sep=";")
+
+  # load system information 
+  systems_df <- read.csv(paste0(data_dir,"/02 Management_CSV/LRB_Systems.csv"), header=TRUE, sep=";")
+
+
+
+
+
+  #=== preprocess wq data =========================================
+
+  # add log_Ecoli
+  Raw.dat$log_Ecoli <- log10(Raw.dat$Ecoli)
+
+  # order df
+  Raw.dat <- Raw.dat[order(Raw.dat$SamplePoint, Raw.dat$DateTime),]
+
+  # add sample point informatoin
+  Raw.dat <- left_join(Raw.dat, SPoint.dat, by="SamplePoint")
+  #rm(SPoint.dat)
+
+  # tranform to longformat
+  lRaw.dat <- Raw.dat %>% 
+                        gather(Parameter, value,
+                               T_lab:log_Ecoli
+                        ) %>%
+                            select(
+                              DateTime,SystemType, FlowDirection, System, SampleType, SamplePoint, dist_axial, Parameter, value
+                            )
+  # remove empty entries and duplicates
+  lRaw.dat <- na.omit(lRaw.dat)
+  lRaw.dat <- distinct(lRaw.dat)
+  rm(Raw.dat)
+
+
+
+
+
+  #=== preprocess flow data =======================================
+
+  lflow_hourly.df <- left_join(lflow_hourly.df, select(systems_df, c(SystemID, FlowDirection)),
+                                by=c("System"="SystemID")
+                      )
+
+  lflow_hourly.df <- na.omit(lflow_hourly.df)
+  lflow_hourly.df <- distinct(lflow_hourly.df)
+
+
+
+
+  lflow_daily.df <- left_join(lflow_daily.df, select(systems_df, c(SystemID, FlowDirection)),
+                              by=c("System"="SystemID")
+                     )
+  lflow_daily.df <- na.omit(lflow_daily.df)
+  lflow_daily.df <- distinct(lflow_daily.df)
+
+
+
+
+  lflow_monthly.df <- left_join(lflow_monthly.df, select(systems_df, c(SystemID, FlowDirection)),
+                                by=c("System"="SystemID")
+                      )
+
+  lflow_monthly.df <- na.omit(lflow_monthly.df)
+  lflow_monthly.df <- distinct(lflow_monthly.df)
+
+
+
+  #=== preprocess weather data
+
+
+  rm(weather_daily.df, weather_monthly.df)
+
+  lweather_daily.df$Date <- as.POSIXct(lweather_daily.df$Date, format="%Y-%m-%d")
+
+  weather_raw.df$RDate_Xlt <- as.POSIXct(weather_raw.df$RDate_Xlt)
+
+
+
+
+
+
+
+
+
+  rm(SPoint.dat)
+
+
 } else {
     data_dir <- "/Data_Example/"
-  }
+
 
 # WQ data
 lRaw.dat <- read.csv(file=paste0(wd,data_dir, "WQ_Data.csv"), header = TRUE, sep = ";", dec = ".")
@@ -63,3 +164,4 @@ weather_raw.df$RDate_Xlt <- as.POSIXct(weather_raw.df$RDate_Xlt, format="%Y-%m-%
 lweather_daily.df$Date <- as.POSIXct(lweather_daily.df$Date, format="%Y-%m-%d")
 lweather_monthly.df$Date <- as.POSIXct(lweather_monthly.df$Date, format="%Y-%m-%d")
 
+}
diff --git a/server.R b/server.R
@@ -118,7 +118,7 @@ function(input, output){
     df3 <- df2_WQ() %>% spread(Parameter,value)
 
     # summary_tabl2(Dataframe, columns range, Factorname, col number of factor to summarize)
-    summary_table2(df3, (which(colnames(df3)=="SamplePoint")+1):length(colnames(df3)),
+    summary_table2(df3, (which(colnames(df3)=="dist_axial")+1):length(colnames(df3)),
                    "SamplePoint", which(colnames(df3)=="SamplePoint"))
   })
 
@@ -151,7 +151,7 @@ function(input, output){
   # filter 
   df_flow <- eventReactive(input$FLOW_goInput, {
 
-    if (is.null(input$FLOW_scaleInput) | is.null(input$FLOW_systemInput) | is.null(input$FLOW_sampletypeInput)) {
+    if (is.null(input$FLOW_scaleInput) | is.null(input$FLOW_systemInput) | is.null(input$FLOW_positionInput)) {
       return(NULL)
     }
 
@@ -164,7 +164,7 @@ function(input, output){
           RDate >= strptime(input$FLOW_dateInput[1], format="%Y-%m-%d"),
           RDate <= strptime(input$FLOW_dateInput[2], format="%Y-%m-%d"),
           System %in% input$FLOW_systemInput,
-          SampleType %in% input$FLOW_sampletypeInput
+          Position %in% input$FLOW_positionInput
         )
     }
     else if (input$FLOW_scaleInput == "Daily Summary") {
@@ -173,7 +173,7 @@ function(input, output){
           RDate >= strptime(input$FLOW_dateInput[1], format="%Y-%m-%d"),
           RDate <= strptime(input$FLOW_dateInput[2], format="%Y-%m-%d"),
           System %in% input$FLOW_systemInput,
-          SampleType %in% input$FLOW_sampletypeInput
+          Position %in% input$FLOW_positionInput
         )
     }
     else {
@@ -183,7 +183,7 @@ function(input, output){
           RDate >= strptime(input$FLOW_dateInput[1], format="%Y-%m-%d"),
           RDate <= strptime(input$FLOW_dateInput[2], format="%Y-%m-%d"),
           System %in% input$FLOW_systemInput,
-          SampleType %in% input$FLOW_sampletypeInput
+          Position %in% input$FLOW_positionInput
         )
 
     }
@@ -194,7 +194,7 @@ function(input, output){
 
   output$FLOW_Plot <- renderPlot({
 
-    if (is.null(input$FLOW_scaleInput) | is.null(input$FLOW_systemInput) | is.null(input$FLOW_sampletypeInput) | is.null(df_flow())) {
+    if (is.null(input$FLOW_scaleInput) | is.null(input$FLOW_systemInput) | is.null(input$FLOW_positionInput) | is.null(df_flow())) {
       return(NULL)
     }
 
@@ -204,12 +204,12 @@ function(input, output){
       facet_var_flow <- "System~." 
       # colorInp <- "System"
 
-      ts_plot_flow <- ggplot(data = na.omit(df_flow()), aes(x=RDate, y=value, color=SampleType)) +
+      ts_plot_flow <- ggplot(data = na.omit(df_flow()), aes(x=RDate, y=value, color=Position)) +
         geom_line()+geom_point() + facet_grid(facet_var_flow, scales = "free_y") + theme_bw() + labs(x="")
     }
 
     if (input$FLOW_facetInput == "In/Out") {
-      facet_var_flow <- ".~SampleType"
+      facet_var_flow <- ".~Position"
       #colorInp <- "SamplePoint"
 
       ts_plot_flow <- ggplot(data = na.omit(df_flow()), aes(x=RDate, y=value, color=System)) +
@@ -219,7 +219,7 @@ function(input, output){
     if (input$FLOW_facetInput == "no multiple Plots") {
       facet_var_flow <- ".~."
       #colorInp <- ""
-      ts_plot_flow <- ggplot(data = na.omit(df_flow()), aes(x=RDate, y=value, color=SampleType, shape=System)) +
+      ts_plot_flow <- ggplot(data = na.omit(df_flow()), aes(x=RDate, y=value, color=Position, shape=System)) +
         geom_line()+geom_point()+ theme_bw() + labs(x="")#+ labs(title="")
     }
 
@@ -249,7 +249,7 @@ function(input, output){
       return(NULL)
     }
 
-    df2 <- select(df_flow(), c(RDate, System, SampleType, value)) %>% spread(SampleType,value)
+    df2 <- select(df_flow(), c(RDate, System, Position, value)) %>% spread(Position,value)
 
     # summary_tabl2(Dataframe, columns range, Factorname, col number of factor to summarize)
     summary_table2(df2, (which(colnames(df2)=="System")+1):length(colnames(df2)),
@@ -324,6 +324,8 @@ function(input, output){
       colnames(lweather_raw.df)[1] <- "Date"
       lweather_raw.df$Date <- as.POSIXct(strptime(lweather_raw.df$Date, format = "%d. %m. %Y %R"))
 
+      rm(df1)
+
       #subset
       lweather_raw.df %>%
         filter(

diff --git a/ui.R b/ui.R
@@ -5,10 +5,11 @@ navbarPage("LRB Research Data",
           sidebarLayout(
             sidebarPanel("",
                          dateRangeInput(inputId = "WQ_dateInput", label = "Date",
-                                        start = "2017-01-01",
-                                        end = "2017-12-31",
-                                        min = "2017-01-01",
-                                        max = "2017-12-31",
+                                        start = substr((as.character(min(lRaw.dat$DateTime, na.rm = TRUE))),1,10),
+                                        end = substr((as.character(max(lRaw.dat$DateTime, na.rm = TRUE))),1,10),
+                                        #end = "2017-12-31",
+                                        min = substr((as.character(min(lRaw.dat$DateTime, na.rm = TRUE))),1,10),
+                                        max = substr((as.character(max(lRaw.dat$DateTime, na.rm = TRUE))),1,10),,
                                         format = "yyyy-mm-dd",
                                         startview = "month",
                                         weekstart = 1),
@@ -51,7 +52,7 @@ navbarPage("LRB Research Data",
           sidebarLayout(
             sidebarPanel("",
                          dateRangeInput(inputId = "FLOW_dateInput", label = "Date",
-                                        start = "2017-01-01",
+                                        start = substr((as.character(min(lflow_daily.df$RDate, na.rm = TRUE))),1,10),
                                         end = substr((as.character(max(lflow_daily.df$RDate, na.rm = TRUE))),1,10),
                                         min = substr((as.character(min(lflow_daily.df$RDate, na.rm = TRUE))),1,10),
                                         max = substr((as.character(max(lflow_daily.df$RDate, na.rm = TRUE))),1,10),
@@ -68,9 +69,9 @@ navbarPage("LRB Research Data",
                                             choiceNames =  as.list(as.character(sort(unique(lflow_daily.df$System)))),
                                             choiceValues = as.list(as.character(sort(unique(lflow_daily.df$System))))),
 
-                         checkboxGroupInput(inputId = "FLOW_sampletypeInput", label = "Sample Type",
-                                            choiceNames =  as.list(as.character(sort(unique(lflow_daily.df$SampleType)))),
-                                            choiceValues = as.list(as.character(sort(unique(lflow_daily.df$SampleType))))),
+                         checkboxGroupInput(inputId = "FLOW_positionInput", label = "Position",
+                                            choiceNames =  as.list(as.character(sort(unique(lflow_daily.df$Position)))),
+                                            choiceValues = as.list(as.character(sort(unique(lflow_daily.df$Position))))),
 
                          radioButtons(inputId = "FLOW_facetInput",
                                       label = "Multiple Plots.",
@@ -109,7 +110,7 @@ navbarPage("LRB Research Data",
           sidebarLayout(
             sidebarPanel("",
                          dateRangeInput(inputId = "WEATHER_dateInput", label = "Date",
-                                        start = "2017-01-01",
+                                        start = substr((as.character(min(lweather_daily.df$Date, na.rm = TRUE))),1,10),
                                         end = substr((as.character(max(lweather_daily.df$Date, na.rm = TRUE))),1,10),
                                         min = substr((as.character(min(lweather_daily.df$Date, na.rm = TRUE))),1,10),
                                         max = substr((as.character(max(lweather_daily.df$Date, na.rm = TRUE))),1,10),